From 0bb092eb34dc2439e407d2ba912689c42c307866 Mon Sep 17 00:00:00 2001 From: hukang hwx963878 Date: Sat, 10 Oct 2020 10:18:20 +0800 Subject: [PATCH] update new 410 category modles and update readme and fixed cpp update new 410 category modles on branch r1.0 update new 410 category modles and update readme english cherry-pick r1.0 branch --- .../lite/image_classification/README.en.md | 159 ++++++---- .../lite/image_classification/README.md | 143 +++++---- .../app/src/main/cpp/MindSporeNetnative.cpp | 289 ++++++++++++------ .../lite/object_detection/README.en.md | 2 +- 4 files changed, 373 insertions(+), 220 deletions(-) diff --git a/model_zoo/official/lite/image_classification/README.en.md b/model_zoo/official/lite/image_classification/README.en.md index 6755062d6ec..9f9855b0392 100644 --- a/model_zoo/official/lite/image_classification/README.en.md +++ b/model_zoo/official/lite/image_classification/README.en.md @@ -1,17 +1,17 @@ -## Demo_image_classification +# Demo of Image Classification The following describes how to use the MindSpore Lite C++ APIs (Android JNIs) and MindSpore Lite image classification models to perform on-device inference, classify the content captured by a device camera, and display the most possible classification result on the application's image preview screen. -### 运行依赖 +### Running Dependencies - Android Studio 3.2 or later (Android 4.0 or later is recommended.) - Native development kit (NDK) 21.3 - CMake 3.10.2 [CMake](https://cmake.org/download) - Android software development kit (SDK) 26 or later -- JDK 1.8 or later [JDK]( https://www.oracle.com/downloads/otn-pub/java/JDK/) +- JDK 1.8 or later -### 构建与运行 +### Building and Running 1. Load the sample source code to Android Studio and install the corresponding SDK. (After the SDK version is specified, Android Studio automatically installs the SDK.) @@ -35,9 +35,7 @@ The following describes how to use the MindSpore Lite C++ APIs (Android JNIs) an The mobile phone needs to be turn on "USB debugging mode" before Android Studio can recognize the mobile phone. Huawei mobile phones generally turn on "USB debugging model" in Settings > system and update > developer Options > USB debugging. -3. 在Android设备上，点击“继续安装”，安装完即可查看到设备摄像头捕获的内容和推理结果。 - - Continue the installation on the Android device. After the installation is complete, you can view the content captured by a camera and the inference result. +3. Continue the installation on the Android device. After the installation is complete, you can view the content captured by a camera and the inference result. ![result](images/app_result.jpg) @@ -60,7 +58,7 @@ app │ | └── MindSporeNetnative.h # header file │ | │ ├── java # application code at the Java layer -│ │ └── com.huawei.himindsporedemo +│ │ └── com.mindspore.himindsporedemo │ │ ├── gallery.classify # implementation related to image processing and MindSpore JNI calling │ │ │ └── ... │ │ └── widget # implementation related to camera enabling and drawing @@ -190,31 +188,47 @@ The inference code process is as follows. For details about the complete code, s Convert the image data to be detected into the Tensor format of the MindSpore model. ```cpp - // Convert the Bitmap image passed in from the JAVA layer to Mat for OpenCV processing - BitmapToMat(env, srcBitmap, matImageSrc); - // Processing such as zooming the picture size. - matImgPreprocessed = PreProcessImageData(matImageSrc); + if (!BitmapToLiteMat(env, srcBitmap, &lite_mat_bgr)) { + MS_PRINT("BitmapToLiteMat error"); + return NULL; + } + if (!PreProcessImageData(lite_mat_bgr, &lite_norm_mat_cut)) { + MS_PRINT("PreProcessImageData error"); + return NULL; + } - ImgDims inputDims; - inputDims.channel = matImgPreprocessed.channels(); - inputDims.width = matImgPreprocessed.cols; - inputDims.height = matImgPreprocessed.rows; - float *dataHWC = new float[inputDims.channel * inputDims.width * inputDims.height] + ImgDims inputDims; + inputDims.channel = lite_norm_mat_cut.channel_; + inputDims.width = lite_norm_mat_cut.width_; + inputDims.height = lite_norm_mat_cut.height_; - // Copy the image data to be detected to the dataHWC array. - // The dataHWC[image_size] array here is the intermediate variable of the input MindSpore model tensor. - float *ptrTmp = reinterpret_cast(matImgPreprocessed.data); - for(int i = 0; i < inputDims.channel * inputDims.width * inputDims.height; i++){ - dataHWC[i] = ptrTmp[i]; - } + // Get the mindsore inference environment which created in loadModel(). + void **labelEnv = reinterpret_cast(netEnv); + if (labelEnv == nullptr) { + MS_PRINT("MindSpore error, labelEnv is a nullptr."); + return NULL; + } + MSNetWork *labelNet = static_cast(*labelEnv); - // Assign dataHWC[image_size] to the input tensor variable. - auto msInputs = mSession->GetInputs(); - auto inTensor = msInputs.front(); - memcpy(inTensor->MutableData(), dataHWC, + auto mSession = labelNet->session(); + if (mSession == nullptr) { + MS_PRINT("MindSpore error, Session is a nullptr."); + return NULL; + } + MS_PRINT("MindSpore get session."); + + auto msInputs = mSession->GetInputs(); + if (msInputs.size() == 0) { + MS_PRINT("MindSpore error, msInputs.size() equals 0."); + return NULL; + } + auto inTensor = msInputs.front(); + + float *dataHWC = reinterpret_cast(lite_norm_mat_cut.data_ptr_); + // Copy dataHWC to the model input tensor. + memcpy(inTensor->MutableData(), dataHWC, inputDims.channel * inputDims.width * inputDims.height * sizeof(float)); - delete[] (dataHWC); - ``` + ``` 3. Perform inference on the input tensor based on the model, obtain the output tensor, and perform post-processing. @@ -240,39 +254,56 @@ The inference code process is as follows. For details about the complete code, s - Perform post-processing of the output data. ```cpp - std::string ProcessRunnetResult(std::unordered_map msOutputs, int runnetRet) { - - std::unordered_map::iterator iter; - iter = msOutputs.begin(); - - // The mobilenetv2.ms model output just one branch. - auto outputTensor = iter->second; - int tensorNum = outputTensor->ElementsNum(); - MS_PRINT("Number of tensor elements:%d", tensorNum); - - // Get a pointer to the first score. - float *temp_scores = static_cast(outputTensor->MutableData()); - - float scores[RET_CATEGORY_SUM]; - for (int i = 0; i < RET_CATEGORY_SUM; ++i) { - if (temp_scores[i] > 0.5) { - MS_PRINT("MindSpore scores[%d] : [%f]", i, temp_scores[i]); - } - scores[i] = temp_scores[i]; - } - - // Score for each category. - // Converted to text information that needs to be displayed in the APP. - std::string categoryScore = ""; - for (int i = 0; i < RET_CATEGORY_SUM; ++i) { - categoryScore += labels_name_map[i]; - categoryScore += ":"; - std::string score_str = std::to_string(scores[i]); - categoryScore += score_str; - categoryScore += ";"; - } - return categoryScore; - } - ``` + std::string ProcessRunnetResult(const int RET_CATEGORY_SUM, const char *const labels_name_map[], + std::unordered_map msOutputs) { + // Get the branch of the model output. + // Use iterators to get map elements. + std::unordered_map::iterator iter; + iter = msOutputs.begin(); + // The mobilenetv2.ms model output just one branch. + auto outputTensor = iter->second; + + int tensorNum = outputTensor->ElementsNum(); + MS_PRINT("Number of tensor elements:%d", tensorNum); + + // Get a pointer to the first score. + float *temp_scores = static_cast(outputTensor->MutableData()); + float scores[RET_CATEGORY_SUM]; + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + scores[i] = temp_scores[i]; + } + + float unifiedThre = 0.5; + float probMax = 1.0; + for (size_t i = 0; i < RET_CATEGORY_SUM; ++i) { + float threshold = g_thres_map[i]; + float tmpProb = scores[i]; + if (tmpProb < threshold) { + tmpProb = tmpProb / threshold * unifiedThre; + } else { + tmpProb = (tmpProb - threshold) / (probMax - threshold) * unifiedThre + unifiedThre; + } + scores[i] = tmpProb; + } + + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + if (scores[i] > 0.5) { + MS_PRINT("MindSpore scores[%d] : [%f]", i, scores[i]); + } + } + + // Score for each category. + // Converted to text information that needs to be displayed in the APP. + std::string categoryScore = ""; + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + categoryScore += labels_name_map[i]; + categoryScore += ":"; + std::string score_str = std::to_string(scores[i]); + categoryScore += score_str; + categoryScore += ";"; + } + return categoryScore; + } + + ``` diff --git a/model_zoo/official/lite/image_classification/README.md b/model_zoo/official/lite/image_classification/README.md index 7014ec716db..0bff2867d5d 100644 --- a/model_zoo/official/lite/image_classification/README.md +++ b/model_zoo/official/lite/image_classification/README.md @@ -9,7 +9,7 @@ - NDK 21.3 - CMake 3.10.2 [CMake](https://cmake.org/download) - Android SDK >= 26 -- JDK >= 1.8 [JDK]( https://www.oracle.com/downloads/otn-pub/java/JDK/) +- JDK >= 1.8 ### 构建与运行 @@ -68,7 +68,7 @@ app | | └── MsNetWork.cpp # MindSpre接口封装 │ | │ ├── java # java层应用代码 -│ │ └── com.huawei.himindsporedemo +│ │ └── com.mindspore.himindsporedemo │ │ ├── gallery.classify # 图像处理及MindSpore JNI调用相关实现 │ │ │ └── ... │ │ └── widget # 开启摄像头及绘制相关实现 @@ -200,34 +200,50 @@ target_link_libraries( 2. 将输入图片转换为传入MindSpore模型的Tensor格式。 将待检测图片数据转换为输入MindSpore模型的Tensor。 - + ```cpp - // Convert the Bitmap image passed in from the JAVA layer to Mat for OpenCV processing - BitmapToMat(env, srcBitmap, matImageSrc); - // Processing such as zooming the picture size. - matImgPreprocessed = PreProcessImageData(matImageSrc); - - ImgDims inputDims; - inputDims.channel = matImgPreprocessed.channels(); - inputDims.width = matImgPreprocessed.cols; - inputDims.height = matImgPreprocessed.rows; - float *dataHWC = new float[inputDims.channel * inputDims.width * inputDims.height] - - // Copy the image data to be detected to the dataHWC array. - // The dataHWC[image_size] array here is the intermediate variable of the input MindSpore model tensor. - float *ptrTmp = reinterpret_cast(matImgPreprocessed.data); - for(int i = 0; i < inputDims.channel * inputDims.width * inputDims.height; i++){ - dataHWC[i] = ptrTmp[i]; + if (!BitmapToLiteMat(env, srcBitmap, &lite_mat_bgr)) { + MS_PRINT("BitmapToLiteMat error"); + return NULL; + } + if (!PreProcessImageData(lite_mat_bgr, &lite_norm_mat_cut)) { + MS_PRINT("PreProcessImageData error"); + return NULL; } - // Assign dataHWC[image_size] to the input tensor variable. + ImgDims inputDims; + inputDims.channel = lite_norm_mat_cut.channel_; + inputDims.width = lite_norm_mat_cut.width_; + inputDims.height = lite_norm_mat_cut.height_; + + // Get the mindsore inference environment which created in loadModel(). + void **labelEnv = reinterpret_cast(netEnv); + if (labelEnv == nullptr) { + MS_PRINT("MindSpore error, labelEnv is a nullptr."); + return NULL; + } + MSNetWork *labelNet = static_cast(*labelEnv); + + auto mSession = labelNet->session(); + if (mSession == nullptr) { + MS_PRINT("MindSpore error, Session is a nullptr."); + return NULL; + } + MS_PRINT("MindSpore get session."); + auto msInputs = mSession->GetInputs(); + if (msInputs.size() == 0) { + MS_PRINT("MindSpore error, msInputs.size() equals 0."); + return NULL; + } auto inTensor = msInputs.front(); + + float *dataHWC = reinterpret_cast(lite_norm_mat_cut.data_ptr_); + // Copy dataHWC to the model input tensor. memcpy(inTensor->MutableData(), dataHWC, - inputDims.channel * inputDims.width * inputDims.height * sizeof(float)); - delete[] (dataHWC); - ``` - + inputDims.channel * inputDims.width * inputDims.height * sizeof(float)); + ``` + 3. 对输入Tensor按照模型进行推理，获取输出Tensor，并进行后处理。 - 图执行，端测推理。 @@ -245,45 +261,62 @@ target_link_libraries( auto temp_dat =mSession->GetOutputByTensorName(name); msOutputs.insert(std::pair {name, temp_dat}); } - std::string retStr = ProcessRunnetResult(msOutputs, ret); + std::string resultStr = ProcessRunnetResult(::RET_CATEGORY_SUM, + ::labels_name_map, msOutputs); ``` - 输出数据的后续处理。 ```cpp std::string ProcessRunnetResult(const int RET_CATEGORY_SUM, const char *const labels_name_map[], - std::unordered_map msOutputs) { - // Get the branch of the model output. - // Use iterators to get map elements. - std::unordered_map::iterator iter; - iter = msOutputs.begin(); + std::unordered_map msOutputs) { + // Get the branch of the model output. + // Use iterators to get map elements. + std::unordered_map::iterator iter; + iter = msOutputs.begin(); - // The mobilenetv2.ms model output just one branch. - auto outputTensor = iter->second; + // The mobilenetv2.ms model output just one branch. + auto outputTensor = iter->second; - int tensorNum = outputTensor->ElementsNum(); - MS_PRINT("Number of tensor elements:%d", tensorNum); + int tensorNum = outputTensor->ElementsNum(); + MS_PRINT("Number of tensor elements:%d", tensorNum); - // Get a pointer to the first score. - float *temp_scores = static_cast(outputTensor->MutableData()); + // Get a pointer to the first score. + float *temp_scores = static_cast(outputTensor->MutableData()); + float scores[RET_CATEGORY_SUM]; + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + scores[i] = temp_scores[i]; + } - float scores[RET_CATEGORY_SUM]; - for (int i = 0; i < RET_CATEGORY_SUM; ++i) { - if (temp_scores[i] > 0.5) { - MS_PRINT("MindSpore scores[%d] : [%f]", i, temp_scores[i]); - } - scores[i] = temp_scores[i]; - } - - // Score for each category. - // Converted to text information that needs to be displayed in the APP. - std::string categoryScore = ""; - for (int i = 0; i < RET_CATEGORY_SUM; ++i) { - categoryScore += labels_name_map[i]; - categoryScore += ":"; - std::string score_str = std::to_string(scores[i]); - categoryScore += score_str; - categoryScore += ";"; - } - return categoryScore; + float unifiedThre = 0.5; + float probMax = 1.0; + for (size_t i = 0; i < RET_CATEGORY_SUM; ++i) { + float threshold = g_thres_map[i]; + float tmpProb = scores[i]; + if (tmpProb < threshold) { + tmpProb = tmpProb / threshold * unifiedThre; + } else { + tmpProb = (tmpProb - threshold) / (probMax - threshold) * unifiedThre + unifiedThre; + } + scores[i] = tmpProb; } + + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + if (scores[i] > 0.5) { + MS_PRINT("MindSpore scores[%d] : [%f]", i, scores[i]); + } + } + + // Score for each category. + // Converted to text information that needs to be displayed in the APP. + std::string categoryScore = ""; + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + categoryScore += labels_name_map[i]; + categoryScore += ":"; + std::string score_str = std::to_string(scores[i]); + categoryScore += score_str; + categoryScore += ";"; + } + return categoryScore; + } + ``` diff --git a/model_zoo/official/lite/image_classification/app/src/main/cpp/MindSporeNetnative.cpp b/model_zoo/official/lite/image_classification/app/src/main/cpp/MindSporeNetnative.cpp index 4f9542e8fbd..9ebc3eade79 100644 --- a/model_zoo/official/lite/image_classification/app/src/main/cpp/MindSporeNetnative.cpp +++ b/model_zoo/official/lite/image_classification/app/src/main/cpp/MindSporeNetnative.cpp @@ -35,102 +35,176 @@ using mindspore::dataset::LPixelType; using mindspore::dataset::LDataType; #define MS_PRINT(format, ...) __android_log_print(ANDROID_LOG_INFO, "MSJNI", format, ##__VA_ARGS__) -static const int RET_CATEGORY_SUM = 601; + +static const int RET_CATEGORY_SUM = 410; static const char *labels_name_map[RET_CATEGORY_SUM] = { - {"Tortoise"}, {"Container"}, {"Magpie"}, {"Seaturtle"}, {"Football"}, {"Ambulance"}, {"Ladder"}, - {"Toothbrush"}, {"Syringe"}, {"Sink"}, {"Toy"}, {"Organ(MusicalInstrument) "}, {"Cassettedeck"}, - {"Apple"}, {"Humaneye"}, {"Cosmetics"}, {"Paddle"}, {"Snowman"}, {"Beer"}, {"Chopsticks"}, - {"Humanbeard"}, {"Bird"}, {"Parkingmeter"}, {"Trafficlight"}, {"Croissant"}, {"Cucumber"}, - {"Radish"}, {"Towel"}, {"Doll"}, {"Skull"}, {"Washingmachine"}, {"Glove"}, {"Tick"}, {"Belt"}, - {"Sunglasses"}, {"Banjo"}, {"Cart"}, {"Ball"}, {"Backpack"}, {"Bicycle"}, {"Homeappliance"}, - {"Centipede"}, {"Boat"}, {"Surfboard"}, {"Boot"}, {"Headphones"}, {"Hotdog"}, {"Shorts"}, - {"Fastfood"}, {"Bus"}, {"Boy "}, {"Screwdriver"}, {"Bicyclewheel"}, {"Barge"}, {"Laptop"}, - {"Miniskirt"}, {"Drill(Tool)"}, {"Dress"}, {"Bear"}, {"Waffle"}, {"Pancake"}, {"Brownbear"}, - {"Woodpecker"}, {"Bluejay"}, {"Pretzel"}, {"Bagel"}, {"Tower"}, {"Teapot"}, {"Person"}, - {"Bowandarrow"}, {"Swimwear"}, {"Beehive"}, {"Brassiere"}, {"Bee"}, {"Bat(Animal)"}, - {"Starfish"}, {"Popcorn"}, {"Burrito"}, {"Chainsaw"}, {"Balloon"}, {"Wrench"}, {"Tent"}, - {"Vehicleregistrationplate"}, {"Lantern"}, {"Toaster"}, {"Flashlight"}, {"Billboard"}, - {"Tiara"}, {"Limousine"}, {"Necklace"}, {"Carnivore"}, {"Scissors"}, {"Stairs"}, - {"Computerkeyboard"}, {"Printer"}, {"Trafficsign"}, {"Chair"}, {"Shirt"}, {"Poster"}, - {"Cheese"}, {"Sock"}, {"Firehydrant"}, {"Landvehicle"}, {"Earrings"}, {"Tie"}, {"Watercraft"}, - {"Cabinetry"}, {"Suitcase"}, {"Muffin"}, {"Bidet"}, {"Snack"}, {"Snowmobile"}, {"Clock"}, - {"Medicalequipment"}, {"Cattle"}, {"Cello"}, {"Jetski"}, {"Camel"}, {"Coat"}, {"Suit"}, - {"Desk"}, {"Cat"}, {"Bronzesculpture"}, {"Juice"}, {"Gondola"}, {"Beetle"}, {"Cannon"}, - {"Computermouse"}, {"Cookie"}, {"Officebuilding"}, {"Fountain"}, {"Coin"}, {"Calculator"}, - {"Cocktail"}, {"Computermonitor"}, {"Box"}, {"Stapler"}, {"Christmastree"}, {"Cowboyhat"}, - {"Hikingequipment"}, {"Studiocouch"}, {"Drum"}, {"Dessert"}, {"Winerack"}, {"Drink"}, - {"Zucchini"}, {"Ladle"}, {"Humanmouth"}, {"DairyProduct"}, {"Dice"}, {"Oven"}, {"Dinosaur"}, - {"Ratchet(Device)"}, {"Couch"}, {"Cricketball"}, {"Wintermelon"}, {"Spatula"}, {"Whiteboard"}, - {"Pencilsharpener"}, {"Door"}, {"Hat"}, {"Shower"}, {"Eraser"}, {"Fedora"}, {"Guacamole"}, - {"Dagger"}, {"Scarf"}, {"Dolphin"}, {"Sombrero"}, {"Tincan"}, {"Mug"}, {"Tap"}, {"Harborseal"}, - {"Stretcher"}, {"Canopener"}, {"Goggles"}, {"Humanbody"}, {"Rollerskates"}, {"Coffeecup"}, - {"Cuttingboard"}, {"Blender"}, {"Plumbingfixture"}, {"Stopsign"}, {"Officesupplies"}, - {"Volleyball(Ball)"}, {"Vase"}, {"Slowcooker"}, {"Wardrobe"}, {"Coffee"}, {"Whisk"}, - {"Papertowel"}, {"Personalcare"}, {"Food"}, {"Sunhat"}, {"Treehouse"}, {"Flyingdisc"}, - {"Skirt"}, {"Gasstove"}, {"Saltandpeppershakers"}, {"Mechanicalfan"}, {"Facepowder"}, {"Fax"}, - {"Fruit"}, {"Frenchfries"}, {"Nightstand"}, {"Barrel"}, {"Kite"}, {"Tart"}, {"Treadmill"}, - {"Fox"}, {"Flag"}, {"Frenchhorn"}, {"Windowblind"}, {"Humanfoot"}, {"Golfcart"}, {"Jacket"}, - {"Egg(Food)"}, {"Streetlight"}, {"Guitar"}, {"Pillow"}, {"Humanleg"}, {"Isopod"}, {"Grape"}, - {"Humanear"}, {"Powerplugsandsockets"}, {"Panda"}, {"Giraffe"}, {"Woman"}, {"Doorhandle"}, - {"Rhinoceros"}, {"Bathtub"}, {"Goldfish"}, {"Houseplant"}, {"Goat"}, {"Baseballbat"}, - {"Baseballglove"}, {"Mixingbowl"}, {"Marineinvertebrates"}, {"Kitchenutensil"}, {"Lightswitch"}, - {"House"}, {"Horse"}, {"Stationarybicycle"}, {"Hammer"}, {"Ceilingfan"}, {"Sofabed"}, - {"Adhesivetape "}, {"Harp"}, {"Sandal"}, {"Bicyclehelmet"}, {"Saucer"}, {"Harpsichord"}, - {"Humanhair"}, {"Heater"}, {"Harmonica"}, {"Hamster"}, {"Curtain"}, {"Bed"}, {"Kettle"}, - {"Fireplace"}, {"Scale"}, {"Drinkingstraw"}, {"Insect"}, {"Hairdryer"}, {"Kitchenware"}, - {"Indoorrower"}, {"Invertebrate"}, {"Foodprocessor"}, {"Bookcase"}, {"Refrigerator"}, - {"Wood-burningstove"}, {"Punchingbag"}, {"Commonfig"}, {"Cocktailshaker"}, {"Jaguar(Animal)"}, - {"Golfball"}, {"Fashionaccessory"}, {"Alarmclock"}, {"Filingcabinet"}, {"Artichoke"}, {"Table"}, - {"Tableware"}, {"Kangaroo"}, {"Koala"}, {"Knife"}, {"Bottle"}, {"Bottleopener"}, {"Lynx"}, - {"Lavender(Plant)"}, {"Lighthouse"}, {"Dumbbell"}, {"Humanhead"}, {"Bowl"}, {"Humidifier"}, - {"Porch"}, {"Lizard"}, {"Billiardtable"}, {"Mammal"}, {"Mouse"}, {"Motorcycle"}, - {"Musicalinstrument"}, {"Swimcap"}, {"Fryingpan"}, {"Snowplow"}, {"Bathroomcabinet"}, - {"Missile"}, {"Bust"}, {"Man"}, {"Waffleiron"}, {"Milk"}, {"Ringbinder"}, {"Plate"}, - {"Mobilephone"}, {"Bakedgoods"}, {"Mushroom"}, {"Crutch"}, {"Pitcher(Container)"}, {"Mirror"}, - {"Personalflotationdevice"}, {"Tabletennisracket"}, {"Pencilcase"}, {"Musicalkeyboard"}, - {"Scoreboard"}, {"Briefcase"}, {"Kitchenknife"}, {"Nail(Construction)"}, {"Tennisball"}, - {"Plasticbag"}, {"Oboe"}, {"Chestofdrawers"}, {"Ostrich"}, {"Piano"}, {"Girl"}, {"Plant"}, - {"Potato"}, {"Hairspray"}, {"Sportsequipment"}, {"Pasta"}, {"Penguin"}, {"Pumpkin"}, {"Pear"}, - {"Infantbed"}, {"Polarbear"}, {"Mixer"}, {"Cupboard"}, {"Jacuzzi"}, {"Pizza"}, {"Digitalclock"}, - {"Pig"}, {"Reptile"}, {"Rifle"}, {"Lipstick"}, {"Skateboard"}, {"Raven"}, {"Highheels"}, - {"Redpanda"}, {"Rose"}, {"Rabbit"}, {"Sculpture"}, {"Saxophone"}, {"Shotgun"}, {"Seafood"}, - {"Submarinesandwich"}, {"Snowboard"}, {"Sword"}, {"Pictureframe"}, {"Sushi"}, {"Loveseat"}, - {"Ski"}, {"Squirrel"}, {"Tripod"}, {"Stethoscope"}, {"Submarine"}, {"Scorpion"}, {"Segway"}, - {"Trainingbench"}, {"Snake"}, {"Coffeetable"}, {"Skyscraper"}, {"Sheep"}, {"Television"}, - {"Trombone"}, {"Tea"}, {"Tank"}, {"Taco"}, {"Telephone"}, {"Torch"}, {"Tiger"}, {"Strawberry"}, - {"Trumpet"}, {"Tree"}, {"Tomato"}, {"Train"}, {"Tool"}, {"Picnicbasket"}, {"Cookingspray"}, - {"Trousers"}, {"Bowlingequipment"}, {"Footballhelmet"}, {"Truck"}, {"Measuringcup"}, - {"Coffeemaker"}, {"Violin"}, {"Vehicle"}, {"Handbag"}, {"Papercutter"}, {"Wine"}, {"Weapon"}, - {"Wheel"}, {"Worm"}, {"Wok"}, {"Whale"}, {"Zebra"}, {"Autopart"}, {"Jug"}, {"Pizzacutter"}, - {"Cream"}, {"Monkey"}, {"Lion"}, {"Bread"}, {"Platter"}, {"Chicken"}, {"Eagle"}, {"Helicopter"}, - {"Owl"}, {"Duck"}, {"Turtle"}, {"Hippopotamus"}, {"Crocodile"}, {"Toilet"}, {"Toiletpaper"}, - {"Squid"}, {"Clothing"}, {"Footwear"}, {"Lemon"}, {"Spider"}, {"Deer"}, {"Frog"}, {"Banana"}, - {"Rocket"}, {"Wineglass"}, {"Countertop"}, {"Tabletcomputer"}, {"Wastecontainer"}, - {"Swimmingpool"}, {"Dog"}, {"Book"}, {"Elephant"}, {"Shark"}, {"Candle"}, {"Leopard"}, {"Axe"}, - {"Handdryer"}, {"Soapdispenser"}, {"Porcupine"}, {"Flower"}, {"Canary"}, {"Cheetah"}, - {"Palmtree"}, {"Hamburger"}, {"Maple"}, {"Building"}, {"Fish"}, {"Lobster"}, - {"GardenAsparagus"}, {"Furniture"}, {"Hedgehog"}, {"Airplane"}, {"Spoon"}, {"Otter"}, {"Bull"}, - {"Oyster"}, {"Horizontalbar"}, {"Conveniencestore"}, {"Bomb"}, {"Bench"}, {"Icecream"}, - {"Caterpillar"}, {"Butterfly"}, {"Parachute"}, {"Orange"}, {"Antelope"}, {"Beaker"}, - {"Mothsandbutterflies"}, {"Window"}, {"Closet"}, {"Castle"}, {"Jellyfish"}, {"Goose"}, {"Mule"}, - {"Swan"}, {"Peach"}, {"Coconut"}, {"Seatbelt"}, {"Raccoon"}, {"Chisel"}, {"Fork"}, {"Lamp"}, - {"Camera"}, {"Squash(Plant)"}, {"Racket"}, {"Humanface"}, {"Humanarm"}, {"Vegetable"}, - {"Diaper"}, {"Unicycle"}, {"Falcon"}, {"Chime"}, {"Snail"}, {"Shellfish"}, {"Cabbage"}, - {"Carrot"}, {"Mango"}, {"Jeans"}, {"Flowerpot"}, {"Pineapple"}, {"Drawer"}, {"Stool"}, - {"Envelope"}, {"Cake"}, {"Dragonfly"}, {"Commonsunflower"}, {"Microwaveoven"}, {"Honeycomb"}, - {"Marinemammal"}, {"Sealion"}, {"Ladybug"}, {"Shelf"}, {"Watch"}, {"Candy"}, {"Salad"}, - {"Parrot"}, {"Handgun"}, {"Sparrow"}, {"Van"}, {"Grinder"}, {"Spicerack"}, {"Lightbulb"}, - {"Cordedphone"}, {"Sportsuniform"}, {"Tennisracket"}, {"Wallclock"}, {"Servingtray"}, - {"Kitchen&diningroomtable"}, {"Dogbed"}, {"Cakestand"}, {"Catfurniture"}, {"Bathroomaccessory"}, - {"Facialtissueholder"}, {"Pressurecooker"}, {"Kitchenappliance"}, {"Tire"}, {"Ruler"}, - {"Luggageandbags"}, {"Microphone"}, {"Broccoli"}, {"Umbrella"}, {"Pastry"}, {"Grapefruit"}, - {"Band-aid"}, {"Animal"}, {"Bellpepper"}, {"Turkey"}, {"Lily"}, {"Pomegranate"}, {"Doughnut"}, - {"Glasses"}, {"Humannose"}, {"Pen"}, {"Ant"}, {"Car"}, {"Aircraft"}, {"Humanhand"}, {"Skunk"}, - {"Teddybear"}, {"Watermelon"}, {"Cantaloupe"}, {"Dishwasher"}, {"Flute"}, {"Balancebeam"}, - {"Sandwich"}, {"Shrimp"}, {"Sewingmachine"}, {"Binoculars"}, {"Raysandskates"}, {"Ipod"}, - {"Accordion"}, {"Willow"}, {"Crab"}, {"Crown"}, {"Seahorse"}, {"Perfume"}, {"Alpaca"}, {"Taxi"}, - {"Canoe"}, {"Remotecontrol"}, {"Wheelchair"}, {"Rugbyball"}, {"Armadillo"}, {"Maracas"}, - {"Helmet"}}; + {"Herd"}, {"Safari"}, {"Bangle"}, {"Cushion"}, {"Countertop"}, + {"Prom"}, {"Branch"}, {"Sports"}, {"Sky"}, {"Community"}, + {"Wheel"}, {"Cola"}, {"Tuxedo"}, {"Flowerpot"}, {"Team"}, + {"Computer"}, {"Unicycle"}, {"Brig"}, {"Aerospace engineering"}, {"Scuba diving"}, + {"Goggles"}, {"Fruit"}, {"Badminton"}, {"Horse"}, {"Sunglasses"}, + {"Fun"}, {"Prairie"}, {"Poster"}, {"Flag"}, {"Speedboat"}, + {"Eyelash"}, {"Veil"}, {"Mobile phone"}, {"Wheelbarrow"}, {"Saucer"}, + {"Leather"}, {"Drawer"}, {"Paper"}, {"Pier"}, {"Waterfowl"}, + {"Tights"}, {"Rickshaw"}, {"Vegetable"}, {"Handrail"}, {"Ice"}, + {"Metal"}, {"Flower"}, {"Wing"}, {"Silverware"}, {"Event"}, + {"Skyline"}, {"Money"}, {"Comics"}, {"Handbag"}, {"Porcelain"}, + {"Rodeo"}, {"Curtain"}, {"Tile"}, {"Human mouth"}, {"Army"}, + {"Menu"}, {"Boat"}, {"Snowboarding"}, {"Cairn terrier"}, {"Net"}, + {"Pasteles"}, {"Cup"}, {"Rugby"}, {"Pho"}, {"Cap"}, + {"Human hair"}, {"Surfing"}, {"Loveseat"}, {"Museum"}, {"Shipwreck"}, + {"Trunk (Tree)"}, {"Plush"}, {"Monochrome"}, {"Volcano"}, {"Rock"}, + {"Pillow"}, {"Presentation"}, {"Nebula"}, {"Subwoofer"}, {"Lake"}, + {"Sledding"}, {"Bangs"}, {"Tablecloth"}, {"Necklace"}, {"Swimwear"}, + {"Standing"}, {"Jeans"}, {"Carnival"}, {"Softball"}, {"Centrepiece"}, + {"Skateboarder"}, {"Cake"}, {"Dragon"}, {"Aurora"}, {"Skiing"}, + {"Bathroom"}, {"Dog"}, {"Needlework"}, {"Umbrella"}, {"Church"}, + {"Fire"}, {"Piano"}, {"Denim"}, {"Bridle"}, {"Cabinetry"}, + {"Lipstick"}, {"Ring"}, {"Television"}, {"Roller"}, {"Seal"}, + {"Concert"}, {"Product"}, {"News"}, {"Fast food"}, {"Horn (Animal)"}, + {"Tattoo"}, {"Bird"}, {"Bridegroom"}, {"Love"}, {"Helmet"}, + {"Dinosaur"}, {"Icing"}, {"Miniature"}, {"Tire"}, {"Toy"}, + {"Icicle"}, {"Jacket"}, {"Coffee"}, {"Mosque"}, {"Rowing"}, + {"Wetsuit"}, {"Camping"}, {"Underwater"}, {"Christmas"}, {"Gelato"}, + {"Whiteboard"}, {"Field"}, {"Ragdoll"}, {"Construction"}, {"Lampshade"}, + {"Palace"}, {"Meal"}, {"Factory"}, {"Cage"}, {"Clipper (Boat)"}, + {"Gymnastics"}, {"Turtle"}, {"Human foot"}, {"Marriage"}, {"Web page"}, + {"Human beard"}, {"Fog"}, {"Wool"}, {"Cappuccino"}, {"Lighthouse"}, + {"Lego"}, {"Sparkler"}, {"Sari"}, {"Model"}, {"Temple"}, + {"Beanie"}, {"Building"}, {"Waterfall"}, {"Penguin"}, {"Cave"}, + {"Stadium"}, {"Smile"}, {"Human hand"}, {"Park"}, {"Desk"}, + {"Shetland sheepdog"}, {"Bar"}, {"Eating"}, {"Neon"}, {"Dalmatian"}, + {"Crocodile"}, {"Wakeboarding"}, {"Longboard"}, {"Road"}, {"Race"}, + {"Kitchen"}, {"Odometer"}, {"Cliff"}, {"Fiction"}, {"School"}, + {"Interaction"}, {"Bullfighting"}, {"Boxer"}, {"Gown"}, {"Aquarium"}, + {"Superhero"}, {"Pie"}, {"Asphalt"}, {"Surfboard"}, {"Cheeseburger"}, + {"Screenshot"}, {"Supper"}, {"Laugh"}, {"Lunch"}, {"Party "}, + {"Glacier"}, {"Bench"}, {"Grandparent"}, {"Sink"}, {"Pomacentridae"}, + {"Blazer"}, {"Brick"}, {"Space"}, {"Backpacking"}, {"Stuffed toy"}, + {"Sushi"}, {"Glitter"}, {"Bonfire"}, {"Castle"}, {"Marathon"}, + {"Pizza"}, {"Beach"}, {"Human ear"}, {"Racing"}, {"Sitting"}, + {"Iceberg"}, {"Shelf"}, {"Vehicle"}, {"Pop music"}, {"Playground"}, + {"Clown"}, {"Car"}, {"Rein"}, {"Fur"}, {"Musician"}, + {"Casino"}, {"Baby"}, {"Alcohol"}, {"Strap"}, {"Reef"}, + {"Balloon"}, {"Outerwear"}, {"Cathedral"}, {"Competition"}, {"Joker"}, + {"Blackboard"}, {"Bunk bed"}, {"Bear"}, {"Moon"}, {"Archery"}, + {"Polo"}, {"River"}, {"Fishing"}, {"Ferris wheel"}, {"Mortarboard"}, + {"Bracelet"}, {"Flesh"}, {"Statue"}, {"Farm"}, {"Desert"}, + {"Chain"}, {"Aircraft"}, {"Textile"}, {"Hot dog"}, {"Knitting"}, + {"Singer"}, {"Juice"}, {"Circus"}, {"Chair"}, {"Musical instrument"}, + {"Room"}, {"Crochet"}, {"Sailboat"}, {"Newspaper"}, {"Santa claus"}, + {"Swamp"}, {"Skyscraper"}, {"Skin"}, {"Rocket"}, {"Aviation"}, + {"Airliner"}, {"Garden"}, {"Ruins"}, {"Storm"}, {"Glasses"}, + {"Balance"}, {"Nail (Body part)"}, {"Rainbow"}, {"Soil "}, {"Vacation "}, + {"Moustache"}, {"Doily"}, {"Food"}, {"Bride "}, {"Cattle"}, + {"Pocket"}, {"Infrastructure"}, {"Train"}, {"Gerbil"}, {"Fireworks"}, + {"Pet"}, {"Dam"}, {"Crew"}, {"Couch"}, {"Bathing"}, + {"Quilting"}, {"Motorcycle"}, {"Butterfly"}, {"Sled"}, {"Watercolor paint"}, + {"Rafting"}, {"Monument"}, {"Lightning"}, {"Sunset"}, {"Bumper"}, + {"Shoe"}, {"Waterskiing"}, {"Sneakers"}, {"Tower"}, {"Insect"}, + {"Pool"}, {"Placemat"}, {"Airplane"}, {"Plant"}, {"Jungle"}, + {"Armrest"}, {"Duck"}, {"Dress"}, {"Tableware"}, {"Petal"}, + {"Bus"}, {"Hanukkah"}, {"Forest"}, {"Hat"}, {"Barn"}, + {"Tubing"}, {"Snorkeling"}, {"Cool"}, {"Cookware and bakeware"}, {"Cycling"}, + {"Swing (Seat)"}, {"Muscle"}, {"Cat"}, {"Skateboard"}, {"Star"}, + {"Toe"}, {"Junk"}, {"Bicycle"}, {"Bedroom"}, {"Person"}, + {"Sand"}, {"Canyon"}, {"Tie"}, {"Twig"}, {"Sphynx"}, + {"Supervillain"}, {"Nightclub"}, {"Ranch"}, {"Pattern"}, {"Shorts"}, + {"Himalayan"}, {"Wall"}, {"Leggings"}, {"Windsurfing"}, {"Deejay"}, + {"Dance"}, {"Van"}, {"Bento"}, {"Sleep"}, {"Wine"}, + {"Picnic"}, {"Leisure"}, {"Dune"}, {"Crowd"}, {"Kayak"}, + {"Ballroom"}, {"Selfie"}, {"Graduation"}, {"Frigate"}, {"Mountain"}, + {"Dude"}, {"Windshield"}, {"Skiff"}, {"Class"}, {"Scarf"}, + {"Bull"}, {"Soccer"}, {"Bag"}, {"Basset hound"}, {"Tractor"}, + {"Swimming"}, {"Running"}, {"Track"}, {"Helicopter"}, {"Pitch"}, + {"Clock"}, {"Song"}, {"Jersey"}, {"Stairs"}, {"Flap"}, + {"Jewellery"}, {"Bridge"}, {"Cuisine"}, {"Bread"}, {"Caving"}, + {"Shell"}, {"Wreath"}, {"Roof"}, {"Cookie"}, {"Canoe"}}; + +static float g_thres_map[RET_CATEGORY_SUM] = { + 0.23, 0.03, 0.10, 0.13, 0.03, + 0.10, 0.06, 0.09, 0.09, 0.05, + 0.01, 0.04, 0.01, 0.27, 0.05, + 0.16, 0.01, 0.16, 0.04, 0.13, + 0.09, 0.18, 0.10, 0.65, 0.08, + 0.04, 0.08, 0.01, 0.05, 0.20, + 0.01, 0.16, 0.10, 0.10, 0.10, + 0.02, 0.24, 0.08, 0.10, 0.53, + 0.07, 0.05, 0.07, 0.27, 0.02, + 0.01, 0.71, 0.01, 0.06, 0.06, + 0.03, 0.96, 0.03, 0.94, 0.05, + 0.03, 0.14, 0.09, 0.03, 0.11, + 0.50, 0.16, 0.07, 0.07, 0.06, + 0.07, 0.08, 0.10, 0.29, 0.03, + 0.05, 0.11, 0.03, 0.03, 0.03, + 0.01, 0.11, 0.07, 0.03, 0.49, + 0.12, 0.30, 0.10, 0.15, 0.02, + 0.06, 0.17, 0.01, 0.04, 0.07, + 0.06, 0.02, 0.19, 0.20, 0.14, + 0.35, 0.15, 0.01, 0.10, 0.13, + 0.43, 0.11, 0.12, 0.32, 0.01, + 0.22, 0.51, 0.02, 0.04, 0.14, + 0.04, 0.35, 0.35, 0.01, 0.54, + 0.04, 0.02, 0.03, 0.02, 0.38, + 0.13, 0.19, 0.06, 0.01, 0.02, + 0.06, 0.03, 0.04, 0.01, 0.10, + 0.01, 0.07, 0.07, 0.07, 0.33, + 0.08, 0.04, 0.06, 0.07, 0.07, + 0.11, 0.02, 0.32, 0.48, 0.14, + 0.01, 0.01, 0.04, 0.05, 0.04, + 0.16, 0.50, 0.11, 0.03, 0.04, + 0.02, 0.55, 0.17, 0.13, 0.84, + 0.18, 0.03, 0.16, 0.02, 0.06, + 0.03, 0.11, 0.96, 0.36, 0.68, + 0.02, 0.08, 0.02, 0.01, 0.03, + 0.05, 0.14, 0.09, 0.06, 0.03, + 0.20, 0.15, 0.62, 0.03, 0.10, + 0.08, 0.02, 0.02, 0.06, 0.03, + 0.04, 0.01, 0.10, 0.05, 0.04, + 0.02, 0.07, 0.03, 0.32, 0.11, + 0.03, 0.02, 0.03, 0.01, 0.03, + 0.03, 0.25, 0.20, 0.19, 0.03, + 0.11, 0.03, 0.02, 0.03, 0.15, + 0.14, 0.06, 0.11, 0.03, 0.02, + 0.02, 0.52, 0.03, 0.02, 0.02, + 0.02, 0.09, 0.56, 0.01, 0.22, + 0.01, 0.48, 0.14, 0.10, 0.08, + 0.73, 0.39, 0.09, 0.10, 0.85, + 0.31, 0.03, 0.05, 0.01, 0.01, + 0.01, 0.10, 0.28, 0.02, 0.03, + 0.04, 0.03, 0.07, 0.14, 0.20, + 0.10, 0.01, 0.05, 0.37, 0.12, + 0.04, 0.44, 0.04, 0.26, 0.08, + 0.07, 0.27, 0.10, 0.03, 0.01, + 0.03, 0.16, 0.41, 0.16, 0.34, + 0.04, 0.30, 0.04, 0.05, 0.18, + 0.33, 0.03, 0.21, 0.03, 0.04, + 0.22, 0.01, 0.04, 0.02, 0.01, + 0.06, 0.02, 0.08, 0.87, 0.11, + 0.15, 0.05, 0.14, 0.09, 0.08, + 0.22, 0.09, 0.07, 0.06, 0.06, + 0.05, 0.43, 0.70, 0.03, 0.07, + 0.06, 0.07, 0.14, 0.04, 0.01, + 0.03, 0.05, 0.65, 0.06, 0.04, + 0.23, 0.06, 0.75, 0.10, 0.01, + 0.63, 0.41, 0.09, 0.01, 0.01, + 0.18, 0.10, 0.03, 0.01, 0.05, + 0.13, 0.18, 0.03, 0.23, 0.01, + 0.04, 0.03, 0.38, 0.90, 0.21, + 0.18, 0.10, 0.48, 0.08, 0.46, + 0.03, 0.01, 0.02, 0.03, 0.10, + 0.01, 0.09, 0.01, 0.01, 0.01, + 0.10, 0.41, 0.01, 0.06, 0.75, + 0.08, 0.01, 0.01, 0.08, 0.21, + 0.06, 0.02, 0.05, 0.02, 0.05, + 0.09, 0.12, 0.03, 0.06, 0.11, + 0.03, 0.01, 0.01, 0.06, 0.84, + 0.04, 0.81, 0.39, 0.02, 0.29, + 0.77, 0.07, 0.06, 0.22, 0.23, + 0.23, 0.01, 0.02, 0.13, 0.04, + 0.19, 0.04, 0.08, 0.27, 0.09, + 0.06, 0.01, 0.03, 0.21, 0.04, +}; char *CreateLocalModelBuffer(JNIEnv *env, jobject modelBuffer) { jbyte *modelAddr = static_cast(env->GetDirectBufferAddress(modelBuffer)); @@ -160,15 +234,30 @@ std::string ProcessRunnetResult(const int RET_CATEGORY_SUM, const char *const la // Get a pointer to the first score. float *temp_scores = static_cast(outputTensor->MutableData()); - float scores[RET_CATEGORY_SUM]; for (int i = 0; i < RET_CATEGORY_SUM; ++i) { - if (temp_scores[i] > 0.5) { - MS_PRINT("MindSpore scores[%d] : [%f]", i, temp_scores[i]); - } - scores[i] = temp_scores[i]; + scores[i] = temp_scores[i]; } + float unifiedThre = 0.5; + float probMax = 1.0; + for (size_t i = 0; i < RET_CATEGORY_SUM; ++i) { + float threshold = g_thres_map[i]; + float tmpProb = scores[i]; + if (tmpProb < threshold) { + tmpProb = tmpProb / threshold * unifiedThre; + } else { + tmpProb = (tmpProb - threshold) / (probMax - threshold) * unifiedThre + unifiedThre; + } + scores[i] = tmpProb; + } + + for (int i = 0; i < RET_CATEGORY_SUM; ++i) { + if (scores[i] > 0.5) { + MS_PRINT("MindSpore scores[%d] : [%f]", i, scores[i]); + } + } + // Score for each category. // Converted to text information that needs to be displayed in the APP. std::string categoryScore = ""; diff --git a/model_zoo/official/lite/object_detection/README.en.md b/model_zoo/official/lite/object_detection/README.en.md index 6867a3b7667..860212ec16f 100644 --- a/model_zoo/official/lite/object_detection/README.en.md +++ b/model_zoo/official/lite/object_detection/README.en.md @@ -1,4 +1,4 @@ -# demo_object_detection +# Demo of Object Detection The following describes how to use the MindSpore Lite C++ APIs (Android JNIs) and MindSpore Lite object detection models to perform on-device inference, detect the content captured by a device camera, and display the most possible detection result on the application's image preview screen.