forked from mindspore-Ecosystem/mindspore
!9893 [MS][LITE]HiMindSpore add star and scene module
From: @sishuikang Reviewed-by: @zhang_xue_tong,@zhanghaibo5 Signed-off-by: @zhang_xue_tong
This commit is contained in:
Gitee
commit
d0bd77c1ef
|
|
@ -63,6 +63,7 @@ public class SplashActivity extends BaseActivity<MainPresenter> implements MainC
|
|||
|
||||
private static final String CODE_URL = "https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/lite";
|
||||
private static final String HELP_URL = "https://github.com/mindspore-ai/mindspore/issues";
|
||||
private static final String STAR_URL = "https://gitee.com/mindspore/mindspore";
|
||||
|
||||
|
||||
@Override
|
||||
|
|
@ -174,6 +175,16 @@ public class SplashActivity extends BaseActivity<MainPresenter> implements MainC
|
|||
}
|
||||
}
|
||||
|
||||
public void onClickSceneDetection(View view) {
|
||||
if (isAllGranted) {
|
||||
ARouter.getInstance().build("/imageobject/ImageCameraActivity")
|
||||
.withInt("OPEN_TYPE", 3).navigation();
|
||||
} else {
|
||||
requestPermissions();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void onClickPhotoDetection(View view) {
|
||||
if (isAllGranted) {
|
||||
ARouter.getInstance().build("/imageobject/ObjectPhotoActivity").navigation();
|
||||
|
|
@ -223,6 +234,10 @@ public class SplashActivity extends BaseActivity<MainPresenter> implements MainC
|
|||
}
|
||||
|
||||
|
||||
public void onClickStar(View view) {
|
||||
openBrowser(STAR_URL);
|
||||
}
|
||||
|
||||
public void openBrowser(String url) {
|
||||
Intent intent = new Intent();
|
||||
intent.setAction("android.intent.action.VIEW");
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="68dp"
|
||||
android:drawableTop="@drawable/logo"
|
||||
android:drawablePadding="20dp"
|
||||
android:gravity="center_horizontal"
|
||||
android:maxLines="1"
|
||||
|
|
@ -23,32 +22,35 @@
|
|||
app:layout_constraintEnd_toEndOf="parent"
|
||||
app:layout_constraintHorizontal_bias="0.0"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintTop_toTopOf="parent" />
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
android:drawableTop="@drawable/logo"
|
||||
/>
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_image"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:layout_marginLeft="20dp"
|
||||
android:layout_marginStart="20dp"
|
||||
android:layout_marginTop="30dp"
|
||||
android:background="@color/gray_btn"
|
||||
android:drawableStart="@drawable/btn_image"
|
||||
android:drawablePadding="5dp"
|
||||
android:gravity="left|center_vertical"
|
||||
android:onClick="onClickImage"
|
||||
android:paddingLeft="4dp"
|
||||
android:paddingStart="4dp"
|
||||
android:text="@string/title_image"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="12sp"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintTop_toBottomOf="@+id/title"
|
||||
app:layout_constraintWidth_percent="0.43" />
|
||||
app:layout_constraintWidth_percent="0.43"
|
||||
tools:ignore="RtlHardcoded,RtlSymmetry" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_image_garbage"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:layout_marginRight="20dp"
|
||||
android:layout_marginEnd="20dp"
|
||||
android:background="@color/gray_btn"
|
||||
android:drawableStart="@drawable/btn_other"
|
||||
android:drawablePadding="5dp"
|
||||
|
|
@ -152,16 +154,16 @@
|
|||
app:layout_constraintWidth_percent="0.43" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_source"
|
||||
android:id="@+id/btn_scene"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:background="@color/gray_btn"
|
||||
android:drawableStart="@drawable/btn_code"
|
||||
android:drawablePadding="5dp"
|
||||
android:gravity="left|center_vertical"
|
||||
android:onClick="onClickSouceCode"
|
||||
android:onClick="onClickSceneDetection"
|
||||
android:paddingLeft="4dp"
|
||||
android:text="@string/title_source"
|
||||
android:text="@string/title_scene"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="12sp"
|
||||
app:layout_constraintBottom_toBottomOf="@+id/btn_segmentation"
|
||||
|
|
@ -169,22 +171,59 @@
|
|||
app:layout_constraintWidth_percent="0.43" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_help"
|
||||
android:layout_marginTop="16dp"
|
||||
android:id="@+id/btn_source"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:layout_marginTop="16dp"
|
||||
android:background="@color/gray_btn"
|
||||
android:drawableStart="@drawable/btn_help"
|
||||
android:drawablePadding="5dp"
|
||||
android:background="@color/gray_light_btn"
|
||||
android:drawableStart="@drawable/btn_code"
|
||||
android:drawablePadding="8dp"
|
||||
android:gravity="left|center_vertical"
|
||||
android:onClick="onClickHelp"
|
||||
android:paddingLeft="4dp"
|
||||
android:text="@string/title_help"
|
||||
android:onClick="onClickSouceCode"
|
||||
android:paddingStart="10dp"
|
||||
android:text="@string/title_source"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="12sp"
|
||||
app:layout_constraintStart_toStartOf="@+id/btn_segmentation"
|
||||
app:layout_constraintTop_toBottomOf="@+id/btn_segmentation"
|
||||
app:layout_constraintWidth_percent="0.43" />
|
||||
app:layout_constraintWidth_percent="0.28" />
|
||||
|
||||
<Button
|
||||
android:layout_marginStart="10dp"
|
||||
android:id="@+id/btn_star"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:background="@color/gray_light_btn"
|
||||
android:drawableStart="@drawable/btn_commend"
|
||||
android:drawablePadding="8dp"
|
||||
android:gravity="left|center_vertical"
|
||||
android:onClick="onClickStar"
|
||||
android:paddingStart="10dp"
|
||||
android:text="@string/title_star"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="12sp"
|
||||
app:layout_constraintEnd_toEndOf="@+id/btn_scene"
|
||||
app:layout_constraintBottom_toBottomOf="@+id/btn_source"
|
||||
app:layout_constraintWidth_percent="0.28" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_help"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="48dp"
|
||||
android:layout_marginTop="16dp"
|
||||
android:background="@color/gray_light_btn"
|
||||
android:drawableStart="@drawable/btn_help"
|
||||
android:drawablePadding="8dp"
|
||||
android:gravity="left|center_vertical"
|
||||
android:onClick="onClickHelp"
|
||||
android:paddingStart="10dp"
|
||||
android:text="@string/title_help"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="12sp"
|
||||
app:layout_constraintStart_toEndOf="@+id/btn_source"
|
||||
app:layout_constraintBottom_toBottomOf="@+id/btn_source"
|
||||
app:layout_constraintEnd_toStartOf="@+id/btn_star"
|
||||
app:layout_constraintWidth_percent="0.28" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/tv_vision"
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
<color name="black">#000000</color>
|
||||
<color name="gray">#A69D9D</color>
|
||||
<color name="gray_btn">#424242</color>
|
||||
<color name="gray_light_btn">#6C6B6B</color>
|
||||
|
||||
<color name="text_blue">#6DA7FF</color>
|
||||
<color name="text_yellow">#F8E71C</color>
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@
|
|||
<string name="title_object_camera">Camera Detection</string>
|
||||
<string name="title_pose_net">PoseNet</string>
|
||||
<string name="title_style_transfer">Style Transfer</string>
|
||||
<string name="title_source">Source Code</string>
|
||||
<string name="title_help">Help And FeedBack</string>
|
||||
<string name="title_source">Code</string>
|
||||
<string name="title_scene">Scene Detection</string>
|
||||
<string name="title_help">FeedBack</string>
|
||||
<string name="title_star">Star</string>
|
||||
<string name="title_segmentation">Image Segmentation</string>
|
||||
|
||||
<string name="title_photo">Photo</string>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,472 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <jni.h>
|
||||
#include <android/bitmap.h>
|
||||
#include <android/asset_manager_jni.h>
|
||||
#include <android/log.h>
|
||||
#include <utility>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <set>
|
||||
#include "include/errorcode.h"
|
||||
#include "include/ms_tensor.h"
|
||||
#include "SceneMindSporeNetnative.h"
|
||||
#include "MSNetWork.h"
|
||||
#include "lite_cv/lite_mat.h"
|
||||
#include "lite_cv/image_process.h"
|
||||
|
||||
using mindspore::dataset::LiteMat;
|
||||
using mindspore::dataset::LPixelType;
|
||||
using mindspore::dataset::LDataType;
|
||||
#define MS_PRINT(format, ...) __android_log_print(ANDROID_LOG_INFO, "MSJNI", format, ##__VA_ARGS__)
|
||||
|
||||
|
||||
static const int RET_CATEGORY_SUM = 365;
|
||||
static const char *labels_name_map[RET_CATEGORY_SUM] = {"airfield", "airplane_cabin",
|
||||
"airport_terminal", "alcove", "alley",
|
||||
"amphitheater", "amusement_arcade",
|
||||
"amusement_park",
|
||||
"apartment_building/outdoor", "aquarium",
|
||||
"aqueduct", "arcade", "arch",
|
||||
"archaelogical_excavation", "archive",
|
||||
"arena/hockey", "arena/performance",
|
||||
"arena/rodeo", "army_base", "art_gallery",
|
||||
"art_school", "art_studio", "artists_loft",
|
||||
"assembly_line", "athletic_field/outdoor",
|
||||
"atrium/public", "attic", "auditorium",
|
||||
"auto_factory", "auto_showroom",
|
||||
"badlands", "bakery/shop",
|
||||
"balcony/exterior", "balcony/interior",
|
||||
"ball_pit",
|
||||
"ballroom", "bamboo_forest", "bank_vault",
|
||||
"banquet_hall", "bar",
|
||||
"barn", "barndoor", "baseball_field",
|
||||
"basement", "basketball_court/indoor",
|
||||
"bathroom", "bazaar/indoor",
|
||||
"bazaar/outdoor", "beach", "beach_house",
|
||||
"beauty_salon", "bedchamber", "bedroom",
|
||||
"beer_garden", "beer_hall",
|
||||
"berth", "biology_laboratory", "boardwalk",
|
||||
"boat_deck", "boathouse",
|
||||
"bookstore", "booth/indoor",
|
||||
"botanical_garden", "bow_window/indoor",
|
||||
"bowling_alley",
|
||||
"boxing_ring", "bridge", "building_facade",
|
||||
"bullring", "burial_chamber",
|
||||
"bus_interior", "bus_station/indoor",
|
||||
"butchers_shop", "butte", "cabin/outdoor",
|
||||
"cafeteria", "campsite", "campus",
|
||||
"canal/natural", "canal/urban",
|
||||
"candy_store", "canyon", "car_interior",
|
||||
"carrousel", "castle",
|
||||
"catacomb", "cemetery", "chalet",
|
||||
"chemistry_lab", "childs_room",
|
||||
"church/indoor", "church/outdoor",
|
||||
"classroom", "clean_room", "cliff",
|
||||
"closet", "clothing_store", "coast",
|
||||
"cockpit", "coffee_shop",
|
||||
"computer_room", "conference_center",
|
||||
"conference_room", "construction_site",
|
||||
"corn_field",
|
||||
"corral", "corridor", "cottage",
|
||||
"courthouse", "courtyard",
|
||||
"creek", "crevasse", "crosswalk", "dam",
|
||||
"delicatessen",
|
||||
"department_store", "desert/sand",
|
||||
"desert/vegetation", "desert_road",
|
||||
"diner/outdoor",
|
||||
"dining_hall", "dining_room", "discotheque",
|
||||
"doorway/outdoor", "dorm_room",
|
||||
"downtown", "dressing_room", "driveway",
|
||||
"drugstore", "elevator/door",
|
||||
"elevator_lobby", "elevator_shaft",
|
||||
"embassy", "engine_room", "entrance_hall",
|
||||
"escalator/indoor", "excavation",
|
||||
"fabric_store", "farm",
|
||||
"fastfood_restaurant",
|
||||
"field/cultivated", "field/wild",
|
||||
"field_road", "fire_escape", "fire_station",
|
||||
"fishpond", "flea_market/indoor",
|
||||
"florist_shop/indoor", "food_court",
|
||||
"football_field",
|
||||
"forest/broadleaf", "forest_path",
|
||||
"forest_road", "formal_garden", "fountain",
|
||||
"galley", "garage/indoor", "garage/outdoor",
|
||||
"gas_station", "gazebo/exterior",
|
||||
"general_store/indoor",
|
||||
"general_store/outdoor", "gift_shop",
|
||||
"glacier", "golf_course",
|
||||
"greenhouse/indoor", "greenhouse/outdoor",
|
||||
"grotto", "gymnasium/indoor",
|
||||
"hangar/indoor",
|
||||
"hangar/outdoor", "harbor",
|
||||
"hardware_store", "hayfield", "heliport",
|
||||
"highway", "home_office", "home_theater",
|
||||
"hospital", "hospital_room",
|
||||
"hot_spring", "hotel/outdoor", "hotel_room",
|
||||
"house", "hunting_lodge/outdoor",
|
||||
"ice_cream_parlor", "ice_floe", "ice_shelf",
|
||||
"ice_skating_rink/indoor",
|
||||
"ice_skating_rink/outdoor",
|
||||
"iceberg", "igloo", "industrial_area",
|
||||
"inn/outdoor", "islet",
|
||||
"jacuzzi/indoor", "jail_cell",
|
||||
"japanese_garden", "jewelry_shop",
|
||||
"junkyard",
|
||||
"kasbah", "kennel/outdoor",
|
||||
"kindergarden_classroom", "kitchen",
|
||||
"lagoon",
|
||||
"lake/natural", "landfill", "landing_deck",
|
||||
"laundromat", "lawn",
|
||||
"lecture_room", "legislative_chamber",
|
||||
"library/indoor", "library/outdoor",
|
||||
"lighthouse",
|
||||
"living_room", "loading_dock", "lobby",
|
||||
"lock_chamber", "locker_room",
|
||||
"mansion", "manufactured_home",
|
||||
"market/indoor", "market/outdoor", "marsh",
|
||||
"martial_arts_gym", "mausoleum", "medina",
|
||||
"mezzanine", "moat/water",
|
||||
"mosque/outdoor", "motel", "mountain",
|
||||
"mountain_path", "mountain_snowy",
|
||||
"movie_theater/indoor", "museum/indoor",
|
||||
"museum/outdoor", "music_studio",
|
||||
"natural_history_museum",
|
||||
"nursery", "nursing_home", "oast_house",
|
||||
"ocean", "office",
|
||||
"office_building", "office_cubicles",
|
||||
"oilrig", "operating_room", "orchard",
|
||||
"orchestra_pit", "pagoda", "palace",
|
||||
"pantry", "park",
|
||||
"parking_garage/indoor",
|
||||
"parking_garage/outdoor", "parking_lot",
|
||||
"pasture", "patio",
|
||||
"pavilion", "pet_shop", "pharmacy",
|
||||
"phone_booth", "physics_laboratory",
|
||||
"picnic_area", "pier", "pizzeria",
|
||||
"playground", "playroom",
|
||||
"plaza", "pond", "porch", "promenade",
|
||||
"pub/indoor",
|
||||
"racecourse", "raceway", "raft",
|
||||
"railroad_track", "rainforest",
|
||||
"reception", "recreation_room",
|
||||
"repair_shop", "residential_neighborhood",
|
||||
"restaurant",
|
||||
"restaurant_kitchen", "restaurant_patio",
|
||||
"rice_paddy", "river", "rock_arch",
|
||||
"roof_garden", "rope_bridge", "ruin",
|
||||
"runway", "sandbox",
|
||||
"sauna", "schoolhouse", "science_museum",
|
||||
"server_room", "shed",
|
||||
"shoe_shop", "shopfront",
|
||||
"shopping_mall/indoor", "shower",
|
||||
"ski_resort",
|
||||
"ski_slope", "sky", "skyscraper", "slum",
|
||||
"snowfield",
|
||||
"soccer_field", "stable",
|
||||
"stadium/baseball", "stadium/football",
|
||||
"stadium/soccer",
|
||||
"stage/indoor", "stage/outdoor",
|
||||
"staircase", "storage_room", "street",
|
||||
"subway_station/platform", "supermarket",
|
||||
"sushi_bar", "swamp", "swimming_hole",
|
||||
"swimming_pool/indoor",
|
||||
"swimming_pool/outdoor",
|
||||
"synagogue/outdoor", "television_room",
|
||||
"television_studio",
|
||||
"temple/asia", "throne_room",
|
||||
"ticket_booth", "topiary_garden", "tower",
|
||||
"toyshop", "train_interior",
|
||||
"train_station/platform", "tree_farm",
|
||||
"tree_house",
|
||||
"trench", "tundra", "underwater/ocean_deep",
|
||||
"utility_room", "valley",
|
||||
"vegetable_garden", "veterinarians_office",
|
||||
"viaduct", "village", "vineyard",
|
||||
"volcano", "volleyball_court/outdoor",
|
||||
"waiting_room", "water_park", "water_tower",
|
||||
"waterfall", "watering_hole", "wave",
|
||||
"wet_bar", "wheat_field",
|
||||
"wind_farm", "windmill", "yard",
|
||||
"youth_hostel", "zen_garden"};
|
||||
|
||||
|
||||
char *SceneCreateLocalModelBuffer(JNIEnv *env, jobject modelBuffer) {
|
||||
jbyte *modelAddr = static_cast<jbyte *>(env->GetDirectBufferAddress(modelBuffer));
|
||||
int modelLen = static_cast<int>(env->GetDirectBufferCapacity(modelBuffer));
|
||||
char *buffer(new char[modelLen]);
|
||||
memcpy(buffer, modelAddr, modelLen);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* To process the result of mindspore inference.
|
||||
* @param msOutputs
|
||||
* @return
|
||||
*/
|
||||
std::string SceneProcessRunnetResult(const int RET_CATEGORY_SUM, const char *const labels_name_map[],
|
||||
std::unordered_map<std::string, mindspore::tensor::MSTensor *> msOutputs) {
|
||||
// Get the branch of the model output.
|
||||
// Use iterators to get map elements.
|
||||
std::unordered_map<std::string, mindspore::tensor::MSTensor *>::iterator iter;
|
||||
iter = msOutputs.begin();
|
||||
|
||||
// The mobilenetv2.ms model output just one branch.
|
||||
auto outputTensor = iter->second;
|
||||
|
||||
int tensorNum = outputTensor->ElementsNum();
|
||||
MS_PRINT("Number of tensor elements:%d", tensorNum);
|
||||
|
||||
// Get a pointer to the first score.
|
||||
float *temp_scores = static_cast<float *>(outputTensor->MutableData());
|
||||
|
||||
// float scores[RET_CATEGORY_SUM];
|
||||
float scores = temp_scores[0];
|
||||
int cat_loc = 0;
|
||||
for (int i = 0; i < RET_CATEGORY_SUM; ++i) {
|
||||
if (scores < temp_scores[i]) {
|
||||
scores = temp_scores[i];
|
||||
cat_loc = i;
|
||||
}
|
||||
if (temp_scores[i] > 0.5) {
|
||||
MS_PRINT("MindSpore scores[%d] : [%f]", i, temp_scores[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Score for each category.
|
||||
// Converted to text information that needs to be displayed in the APP.
|
||||
std::string categoryScore = "";
|
||||
categoryScore += labels_name_map[cat_loc];
|
||||
categoryScore += ":";
|
||||
std::string score_str = std::to_string(scores);
|
||||
categoryScore += score_str;
|
||||
return categoryScore;
|
||||
}
|
||||
|
||||
bool SceneBitmapToLiteMat(JNIEnv *env, const jobject &srcBitmap, LiteMat *lite_mat) {
|
||||
bool ret = false;
|
||||
AndroidBitmapInfo info;
|
||||
void *pixels = nullptr;
|
||||
LiteMat &lite_mat_bgr = *lite_mat;
|
||||
AndroidBitmap_getInfo(env, srcBitmap, &info);
|
||||
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
|
||||
MS_PRINT("Image Err, Request RGBA");
|
||||
return false;
|
||||
}
|
||||
AndroidBitmap_lockPixels(env, srcBitmap, &pixels);
|
||||
if (info.stride == info.width * 4) {
|
||||
ret = InitFromPixel(reinterpret_cast<const unsigned char *>(pixels),
|
||||
LPixelType::RGBA2RGB, LDataType::UINT8,
|
||||
info.width, info.height, lite_mat_bgr);
|
||||
if (!ret) {
|
||||
MS_PRINT("Init From RGBA error");
|
||||
}
|
||||
} else {
|
||||
unsigned char *pixels_ptr = new unsigned char[info.width * info.height * 4];
|
||||
unsigned char *ptr = pixels_ptr;
|
||||
unsigned char *data = reinterpret_cast<unsigned char *>(pixels);
|
||||
for (int i = 0; i < info.height; i++) {
|
||||
memcpy(ptr, data, info.width * 4);
|
||||
ptr += info.width * 4;
|
||||
data += info.stride;
|
||||
}
|
||||
ret = InitFromPixel(reinterpret_cast<const unsigned char *>(pixels_ptr),
|
||||
LPixelType::RGBA2RGB, LDataType::UINT8,
|
||||
info.width, info.height, lite_mat_bgr);
|
||||
if (!ret) {
|
||||
MS_PRINT("Init From RGBA error");
|
||||
}
|
||||
delete[] (pixels_ptr);
|
||||
}
|
||||
AndroidBitmap_unlockPixels(env, srcBitmap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ScenePreProcessImageData(const LiteMat &lite_mat_bgr, LiteMat *lite_norm_mat_ptr) {
|
||||
bool ret = false;
|
||||
LiteMat lite_mat_resize;
|
||||
LiteMat &lite_norm_mat_cut = *lite_norm_mat_ptr;
|
||||
ret = ResizeBilinear(lite_mat_bgr, lite_mat_resize, 256, 256);
|
||||
if (!ret) {
|
||||
MS_PRINT("ResizeBilinear error");
|
||||
return false;
|
||||
}
|
||||
LiteMat lite_mat_convert_float;
|
||||
ret = ConvertTo(lite_mat_resize, lite_mat_convert_float, 1.0 / 255.0);
|
||||
if (!ret) {
|
||||
MS_PRINT("ConvertTo error");
|
||||
return false;
|
||||
}
|
||||
LiteMat lite_mat_cut;
|
||||
ret = Crop(lite_mat_convert_float, lite_mat_cut, 16, 16, 224, 224);
|
||||
if (!ret) {
|
||||
MS_PRINT("Crop error");
|
||||
return false;
|
||||
}
|
||||
std::vector<float> means = {0.485, 0.456, 0.406};
|
||||
std::vector<float> stds = {0.229, 0.224, 0.225};
|
||||
SubStractMeanNormalize(lite_mat_cut, lite_norm_mat_cut, means, stds);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* The Java layer reads the model into MappedByteBuffer or ByteBuffer to load the model.
|
||||
*/
|
||||
extern "C"
|
||||
JNIEXPORT jlong JNICALL
|
||||
Java_com_mindspore_imageobject_imageclassification_help_SceneTrackingMobile_loadModel(JNIEnv *env,
|
||||
jobject thiz,
|
||||
jobject model_buffer,
|
||||
jint num_thread) {
|
||||
if (nullptr == model_buffer) {
|
||||
MS_PRINT("error, buffer is nullptr!");
|
||||
return (jlong) nullptr;
|
||||
}
|
||||
jlong bufferLen = env->GetDirectBufferCapacity(model_buffer);
|
||||
if (0 == bufferLen) {
|
||||
MS_PRINT("error, bufferLen is 0!");
|
||||
return (jlong) nullptr;
|
||||
}
|
||||
|
||||
char *modelBuffer = SceneCreateLocalModelBuffer(env, model_buffer);
|
||||
if (modelBuffer == nullptr) {
|
||||
MS_PRINT("modelBuffer create failed!");
|
||||
return (jlong) nullptr;
|
||||
}
|
||||
|
||||
// To create a mindspore network inference environment.
|
||||
void **labelEnv = new void *;
|
||||
MSNetWork *labelNet = new MSNetWork;
|
||||
*labelEnv = labelNet;
|
||||
|
||||
mindspore::lite::Context *context = new mindspore::lite::Context;
|
||||
context->thread_num_ = num_thread;
|
||||
context->device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_ = mindspore::lite::NO_BIND;
|
||||
context->device_list_[0].device_info_.cpu_device_info_.enable_float16_ = false;
|
||||
context->device_list_[0].device_type_ = mindspore::lite::DT_CPU;
|
||||
|
||||
labelNet->CreateSessionMS(modelBuffer, bufferLen, context);
|
||||
delete context;
|
||||
|
||||
if (labelNet->session() == nullptr) {
|
||||
MS_PRINT("MindSpore create session failed!.");
|
||||
delete labelNet;
|
||||
delete labelEnv;
|
||||
return (jlong) nullptr;
|
||||
}
|
||||
|
||||
if (model_buffer != nullptr) {
|
||||
env->DeleteLocalRef(model_buffer);
|
||||
}
|
||||
|
||||
return (jlong) labelEnv;
|
||||
}
|
||||
|
||||
/**
|
||||
* After the inference environment is successfully created,
|
||||
* sending a picture to the model and run inference.
|
||||
*/
|
||||
extern "C" JNIEXPORT jstring JNICALL
|
||||
Java_com_mindspore_imageobject_imageclassification_help_SceneTrackingMobile_runNet(JNIEnv *env, jclass type,
|
||||
jlong netEnv,
|
||||
jobject srcBitmap) {
|
||||
LiteMat lite_mat_bgr, lite_norm_mat_cut;
|
||||
|
||||
if (!SceneBitmapToLiteMat(env, srcBitmap, &lite_mat_bgr)) {
|
||||
MS_PRINT("SceneBitmapToLiteMat error");
|
||||
return NULL;
|
||||
}
|
||||
if (!ScenePreProcessImageData(lite_mat_bgr, &lite_norm_mat_cut)) {
|
||||
MS_PRINT("ScenePreProcessImageData error");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ImgDims inputDims;
|
||||
inputDims.channel = lite_norm_mat_cut.channel_;
|
||||
inputDims.width = lite_norm_mat_cut.width_;
|
||||
inputDims.height = lite_norm_mat_cut.height_;
|
||||
|
||||
// Get the mindsore inference environment which created in loadModel().
|
||||
void **labelEnv = reinterpret_cast<void **>(netEnv);
|
||||
if (labelEnv == nullptr) {
|
||||
MS_PRINT("MindSpore error, labelEnv is a nullptr.");
|
||||
return NULL;
|
||||
}
|
||||
MSNetWork *labelNet = static_cast<MSNetWork *>(*labelEnv);
|
||||
|
||||
auto mSession = labelNet->session();
|
||||
if (mSession == nullptr) {
|
||||
MS_PRINT("MindSpore error, Session is a nullptr.");
|
||||
return NULL;
|
||||
}
|
||||
MS_PRINT("MindSpore get session.");
|
||||
|
||||
auto msInputs = mSession->GetInputs();
|
||||
if (msInputs.size() == 0) {
|
||||
MS_PRINT("MindSpore error, msInputs.size() equals 0.");
|
||||
return NULL;
|
||||
}
|
||||
auto inTensor = msInputs.front();
|
||||
|
||||
float *dataHWC = reinterpret_cast<float *>(lite_norm_mat_cut.data_ptr_);
|
||||
// Copy dataHWC to the model input tensor.
|
||||
memcpy(inTensor->MutableData(), dataHWC,
|
||||
inputDims.channel * inputDims.width * inputDims.height * sizeof(float));
|
||||
|
||||
// After the model and image tensor data is loaded, run inference.
|
||||
auto status = mSession->RunGraph();
|
||||
|
||||
if (status != mindspore::lite::RET_OK) {
|
||||
MS_PRINT("MindSpore run net error.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the mindspore inference results.
|
||||
* Return the map of output node name and MindSpore Lite MSTensor.
|
||||
*/
|
||||
auto names = mSession->GetOutputTensorNames();
|
||||
std::unordered_map<std::string, mindspore::tensor::MSTensor *> msOutputs;
|
||||
for (const auto &name : names) {
|
||||
auto temp_dat = mSession->GetOutputByTensorName(name);
|
||||
msOutputs.insert(std::pair<std::string, mindspore::tensor::MSTensor *>{name, temp_dat});
|
||||
}
|
||||
|
||||
std::string resultStr = SceneProcessRunnetResult(::RET_CATEGORY_SUM,
|
||||
::labels_name_map, msOutputs);
|
||||
|
||||
const char *resultCharData = resultStr.c_str();
|
||||
return (env)->NewStringUTF(resultCharData);
|
||||
}
|
||||
|
||||
extern "C" JNIEXPORT jboolean JNICALL
|
||||
Java_com_mindspore_imageobject_imageclassification_help_SceneTrackingMobile_unloadModel(JNIEnv *env,
|
||||
jclass type,
|
||||
jlong netEnv) {
|
||||
MS_PRINT("MindSpore release net.");
|
||||
void **labelEnv = reinterpret_cast<void **>(netEnv);
|
||||
if (labelEnv == nullptr) {
|
||||
MS_PRINT("MindSpore error, labelEnv is a nullptr.");
|
||||
}
|
||||
MSNetWork *labelNet = static_cast<MSNetWork *>(*labelEnv);
|
||||
|
||||
labelNet->ReleaseNets();
|
||||
|
||||
return (jboolean) true;
|
||||
}
|
||||
|
|
@ -1,19 +1,21 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* <p>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.mindspore.imageobject.track;
|
||||
|
||||
public interface TrackListener {
|
||||
}
|
||||
#ifndef SCENE_MINDSPORE_JNI_HMS_DEBUG_MINDSPORENETNATIVE_H
|
||||
#define SCENE_MINDSPORE_JNI_HMS_DEBUG_MINDSPORENETNATIVE_H
|
||||
|
||||
|
||||
#endif // MINDSPORE_JNI_HMS_DEBUG_MINDSPORENETNATIVE_H
|
||||
|
|
@ -16,6 +16,7 @@
|
|||
package com.mindspore.imageobject.camera;
|
||||
|
||||
import android.Manifest;
|
||||
import android.annotation.SuppressLint;
|
||||
import android.app.Activity;
|
||||
import android.content.Context;
|
||||
import android.content.pm.PackageManager;
|
||||
|
|
@ -52,11 +53,6 @@ import androidx.annotation.NonNull;
|
|||
import androidx.annotation.Nullable;
|
||||
import androidx.core.app.ActivityCompat;
|
||||
|
||||
import com.mindspore.imageobject.imageclassification.help.GarbageTrackingMobile;
|
||||
import com.mindspore.imageobject.imageclassification.help.ImageTrackingMobile;
|
||||
import com.mindspore.imageobject.objectdetection.help.ObjectTrackingMobile;
|
||||
import com.mindspore.imageobject.track.TrackListener;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
|
@ -82,12 +78,6 @@ public class CameraPreview extends TextureView {
|
|||
private static final int STATE_WAITING_NON_PRE_CAPTURE = 3;//Other states
|
||||
private static final int STATE_PICTURE_TAKEN = 4;//Photo finished
|
||||
|
||||
public static final int OPEN_TYPE_IMAGE = 1;
|
||||
public static final int OPEN_TYPE_IMAGE_CUSTOM = 11;
|
||||
public static final int OPEN_TYPE_OBJECT = 2;
|
||||
|
||||
private int openType;
|
||||
|
||||
private int mState = STATE_PREVIEW;
|
||||
private int mRatioWidth = 0, mRatioHeight = 0;
|
||||
private int mSensorOrientation;
|
||||
|
|
@ -110,10 +100,6 @@ public class CameraPreview extends TextureView {
|
|||
private ImageReader mImageReader;
|
||||
private ICameraDataCallBack iCameraDataCallBack;
|
||||
|
||||
private ImageTrackingMobile imageTrackingMobile;
|
||||
private GarbageTrackingMobile garbageTrackingMobile;
|
||||
private ObjectTrackingMobile objectTrackingMobile;
|
||||
|
||||
private boolean isPreBackgroundThreadPause;
|
||||
private boolean isAlive;
|
||||
|
||||
|
|
@ -150,7 +136,7 @@ public class CameraPreview extends TextureView {
|
|||
if (0 == mRatioWidth || 0 == mRatioHeight) {
|
||||
setMeasuredDimension(width, height);
|
||||
} else {
|
||||
if (width < height * mRatioWidth / mRatioHeight) {
|
||||
if (width > height * mRatioWidth / mRatioHeight) {
|
||||
setMeasuredDimension(width, width * mRatioHeight / mRatioWidth);
|
||||
} else {
|
||||
setMeasuredDimension(height * mRatioWidth / mRatioHeight, height);
|
||||
|
|
@ -286,23 +272,9 @@ public class CameraPreview extends TextureView {
|
|||
};
|
||||
|
||||
|
||||
public void onResume(Activity activity, int openType, TrackListener track) {
|
||||
public void onResume(Activity activity) {
|
||||
isAlive = true;
|
||||
this.activity = activity;
|
||||
this.openType = openType;
|
||||
if (OPEN_TYPE_IMAGE == openType) {
|
||||
if (null != track) {
|
||||
imageTrackingMobile = (ImageTrackingMobile) track;
|
||||
}
|
||||
} else if (OPEN_TYPE_IMAGE_CUSTOM == openType) {
|
||||
if (null != track) {
|
||||
garbageTrackingMobile = (GarbageTrackingMobile) track;
|
||||
}
|
||||
} else if (OPEN_TYPE_OBJECT == openType) {
|
||||
if (null != track) {
|
||||
objectTrackingMobile = (ObjectTrackingMobile) track;
|
||||
}
|
||||
}
|
||||
startBackgroundThread();
|
||||
//When activity or fragment onresume(), you can open a camera and start previewing, otherwise, the surface is ready
|
||||
if (this.isAvailable()) {
|
||||
|
|
@ -323,12 +295,10 @@ public class CameraPreview extends TextureView {
|
|||
mBackgroundThread.start();
|
||||
mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
|
||||
|
||||
if (OPEN_TYPE_IMAGE == openType || OPEN_TYPE_OBJECT == openType || OPEN_TYPE_IMAGE_CUSTOM == openType) {
|
||||
mImageBackgroundThread = new HandlerThread("MINDSPORE");
|
||||
mImageBackgroundThread.start();
|
||||
mImageBackgroundHandler = new Handler(mImageBackgroundThread.getLooper());
|
||||
mImageBackgroundHandler.post(classifyRunnable);
|
||||
}
|
||||
mImageBackgroundThread = new HandlerThread("MINDSPORE");
|
||||
mImageBackgroundThread.start();
|
||||
mImageBackgroundHandler = new Handler(mImageBackgroundThread.getLooper());
|
||||
mImageBackgroundHandler.post(classifyRunnable);
|
||||
}
|
||||
|
||||
public boolean isAlive() {
|
||||
|
|
@ -342,25 +312,8 @@ public class CameraPreview extends TextureView {
|
|||
public void run() {
|
||||
synchronized (this) {
|
||||
Bitmap bitmap = getBitmap();
|
||||
if (bitmap != null) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
// The current bitmap performs the sending request identification operation
|
||||
String ret = "";
|
||||
if (OPEN_TYPE_IMAGE == openType) {
|
||||
ret = null == imageTrackingMobile ? "" : imageTrackingMobile.MindSpore_runnet(bitmap);
|
||||
} else if (OPEN_TYPE_IMAGE_CUSTOM == openType) {
|
||||
ret = null == garbageTrackingMobile ? "" : garbageTrackingMobile.MindSpore_runnet(bitmap);
|
||||
} else if (OPEN_TYPE_OBJECT == openType) {
|
||||
ret = null == objectTrackingMobile ? "" : objectTrackingMobile.MindSpore_runnet(bitmap);
|
||||
}
|
||||
long endTime = System.currentTimeMillis();
|
||||
if (mRecognitionDataCallBack != null) {
|
||||
// Interface returns data。
|
||||
mRecognitionDataCallBack.onRecognitionDataCallBack(ret, (endTime - startTime) + "ms ");
|
||||
}
|
||||
if (!bitmap.isRecycled()) {
|
||||
bitmap.recycle();
|
||||
}
|
||||
if (getBitmap() != null && mRecognitionDataCallBack != null) {
|
||||
mRecognitionDataCallBack.onRecognitionBitmapCallBack(bitmap);
|
||||
}
|
||||
if (mImageBackgroundHandler != null && !isPreBackgroundThreadPause) {
|
||||
mImageBackgroundHandler.postDelayed(classifyRunnable, 1000);
|
||||
|
|
@ -377,13 +330,10 @@ public class CameraPreview extends TextureView {
|
|||
mBackgroundThread = null;
|
||||
mBackgroundHandler = null;
|
||||
|
||||
if (OPEN_TYPE_IMAGE == openType || OPEN_TYPE_IMAGE_CUSTOM == openType || OPEN_TYPE_OBJECT == openType) {
|
||||
mImageBackgroundThread.quitSafely();
|
||||
mImageBackgroundThread.join();
|
||||
mImageBackgroundThread = null;
|
||||
mImageBackgroundHandler = null;
|
||||
}
|
||||
|
||||
mImageBackgroundThread.quitSafely();
|
||||
mImageBackgroundThread.join();
|
||||
mImageBackgroundThread = null;
|
||||
mImageBackgroundHandler = null;
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
@ -397,13 +347,11 @@ public class CameraPreview extends TextureView {
|
|||
/**
|
||||
* Data interface returned after identification.
|
||||
*
|
||||
* @param result Recognition result
|
||||
* @param time Response time
|
||||
* @param bitmap Recognition bitmap
|
||||
*/
|
||||
void onRecognitionDataCallBack(String result, String time);
|
||||
void onRecognitionBitmapCallBack(Bitmap bitmap);
|
||||
}
|
||||
|
||||
|
||||
private RecognitionDataCallBack mRecognitionDataCallBack;
|
||||
|
||||
public void addImageRecognitionDataCallBack(RecognitionDataCallBack recognitionDataCallBack) {
|
||||
|
|
@ -413,6 +361,7 @@ public class CameraPreview extends TextureView {
|
|||
/**
|
||||
* Turn on the camera according to the mcameraid
|
||||
*/
|
||||
@SuppressLint("MissingPermission")
|
||||
private void openCamera(int width, int height) {
|
||||
setUpCameraOutputs(width, height);
|
||||
configureTransform(width, height);
|
||||
|
|
@ -730,8 +679,6 @@ public class CameraPreview extends TextureView {
|
|||
}
|
||||
|
||||
public void takePicture() {
|
||||
// SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
|
||||
// String date = simpleDateFormat.format(new Date());
|
||||
mFile = new File(getContext().getExternalFilesDir(null), "temp.jpg");
|
||||
if (mFile.length() > 0) {
|
||||
mFile.delete();
|
||||
|
|
|
|||
|
|
@ -20,15 +20,13 @@ import android.content.Context;
|
|||
import android.graphics.Bitmap;
|
||||
import android.util.Log;
|
||||
|
||||
import com.mindspore.imageobject.track.TrackListener;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Call the MindSpore interface API in the Java layer.
|
||||
*/
|
||||
public class GarbageTrackingMobile implements TrackListener {
|
||||
public class GarbageTrackingMobile {
|
||||
private final static String TAG = "GarbageTrackingMobile";
|
||||
|
||||
static {
|
||||
|
|
|
|||
|
|
@ -20,15 +20,13 @@ import android.content.Context;
|
|||
import android.graphics.Bitmap;
|
||||
import android.util.Log;
|
||||
|
||||
import com.mindspore.imageobject.track.TrackListener;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Call the MindSpore interface API in the Java layer.
|
||||
*/
|
||||
public class ImageTrackingMobile implements TrackListener {
|
||||
public class ImageTrackingMobile {
|
||||
private final static String TAG = "ImageTrackingMobile";
|
||||
|
||||
static {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.mindspore.imageobject.imageclassification.help;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.Bitmap;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Call the MindSpore interface API in the Java layer.
|
||||
*/
|
||||
public class SceneTrackingMobile {
|
||||
private final static String TAG = "TrackingMobile";
|
||||
|
||||
static {
|
||||
try {
|
||||
System.loadLibrary("mlkit-label-MS");
|
||||
Log.i(TAG, "load libiMindSpore.so successfully.");
|
||||
} catch (UnsatisfiedLinkError e) {
|
||||
Log.e(TAG, "UnsatisfiedLinkError " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// The address of the running inference environment.
|
||||
private long netEnv = 0;
|
||||
|
||||
private final Context mActivity;
|
||||
|
||||
public SceneTrackingMobile(Context activity) {
|
||||
this.mActivity = activity;
|
||||
}
|
||||
|
||||
/**
|
||||
* JNI load model and also create model inference environment.
|
||||
*
|
||||
* @param modelBuffer Model buffer.
|
||||
* @param numThread The num of thread.
|
||||
* @return MindSpore Inference environment address.
|
||||
*/
|
||||
public native long loadModel(ByteBuffer modelBuffer, int numThread);
|
||||
|
||||
/**
|
||||
* Running model.
|
||||
*
|
||||
* @param netEnv Inference environment address.
|
||||
* @param img A picture to be inferred.
|
||||
* @return Inference result
|
||||
*/
|
||||
public native String runNet(long netEnv, Bitmap img);
|
||||
|
||||
/**
|
||||
* Unbind model data.
|
||||
*
|
||||
* @param netEnv Inference environment address.
|
||||
* @return Unbound state.
|
||||
*/
|
||||
public native boolean unloadModel(long netEnv);
|
||||
|
||||
/**
|
||||
* The C++ side is encapsulated into a method of the MSNetWorks class
|
||||
*
|
||||
* @param modelPath Model file location
|
||||
* @return Load model file status
|
||||
*/
|
||||
public boolean loadModelFromBuf(String modelPath) {
|
||||
ByteBuffer buffer = loadModelFile(modelPath);
|
||||
netEnv = loadModel(buffer, 2); //numThread's default setting is 2.
|
||||
if (netEnv == 0) { // Loading model failed.
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run MindSpore inference.
|
||||
*/
|
||||
public String MindSpore_runnet(Bitmap img) {
|
||||
String ret_str = runNet(netEnv, img);
|
||||
return ret_str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unload model.
|
||||
* @return true
|
||||
*/
|
||||
public boolean unloadModel() {
|
||||
unloadModel(netEnv);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load model file stream.
|
||||
* @param modelPath Model file path.
|
||||
* @return Model ByteBuffer.
|
||||
*/
|
||||
public ByteBuffer loadModelFile(String modelPath) {
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = mActivity.getAssets().open(modelPath);
|
||||
byte[] bytes = new byte[is.available()];
|
||||
is.read(bytes);
|
||||
return ByteBuffer.allocateDirect(bytes.length).put(bytes);
|
||||
} catch (Exception e) {
|
||||
Log.d("loadModelFile", " Exception occur. ");
|
||||
Log.e(TAG, Log.getStackTraceString(e));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
package com.mindspore.imageobject.imageclassification.ui;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.Color;
|
||||
import android.os.Bundle;
|
||||
import android.text.TextUtils;
|
||||
|
|
@ -24,6 +25,7 @@ import android.view.Gravity;
|
|||
import android.view.View;
|
||||
import android.view.ViewGroup;
|
||||
import android.widget.LinearLayout;
|
||||
import android.widget.RelativeLayout;
|
||||
import android.widget.TextView;
|
||||
|
||||
import androidx.annotation.UiThread;
|
||||
|
|
@ -37,6 +39,8 @@ import com.mindspore.imageobject.camera.CameraPreview;
|
|||
import com.mindspore.imageobject.imageclassification.bean.RecognitionImageBean;
|
||||
import com.mindspore.imageobject.imageclassification.help.GarbageTrackingMobile;
|
||||
import com.mindspore.imageobject.imageclassification.help.ImageTrackingMobile;
|
||||
import com.mindspore.imageobject.imageclassification.help.SceneTrackingMobile;
|
||||
import com.mindspore.imageobject.util.DisplayUtil;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
@ -50,9 +54,12 @@ import java.util.List;
|
|||
@Route(path = "/imageobject/ImageCameraActivity")
|
||||
public class ImageCameraActivity extends AppCompatActivity implements CameraPreview.RecognitionDataCallBack {
|
||||
private static final String TAG = "ImageCameraActivity";
|
||||
public static final int TYPE_DEMO = 1;
|
||||
public static final int TYPE_CUSTOM = 2;
|
||||
public static final int TYPE_IMAGE = 1;
|
||||
public static final int TYPE_GARBAGE = 2;
|
||||
public static final int TYPE_SCENE = 3;
|
||||
|
||||
private static final String IMAGE_SCENE_MS = "model/mobilenetv2.ms";
|
||||
private static final String GARBAGE_MS = "model/garbage_mobilenetv2.ms";
|
||||
@Autowired(name = "OPEN_TYPE")
|
||||
int enterType;
|
||||
|
||||
|
|
@ -62,9 +69,9 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
|
||||
private CameraPreview cameraPreview;
|
||||
|
||||
private ImageTrackingMobile mTrackingMobile;
|
||||
|
||||
private ImageTrackingMobile imageTrackingMobile;
|
||||
private GarbageTrackingMobile garbageTrackingMobile;
|
||||
private SceneTrackingMobile sceneTrackingMobile;
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
|
|
@ -81,17 +88,25 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
}
|
||||
|
||||
private void init() {
|
||||
if (enterType == TYPE_DEMO) {
|
||||
mTrackingMobile = new ImageTrackingMobile(this);
|
||||
String modelPath = "model/mobilenetv2.ms";
|
||||
boolean ret = mTrackingMobile.loadModelFromBuf(modelPath);
|
||||
Log.d(TAG, "Loading model return value: " + ret);
|
||||
} else {
|
||||
boolean ret = false;
|
||||
RelativeLayout.LayoutParams linearParams = (RelativeLayout.LayoutParams) bottomLayout.getLayoutParams();
|
||||
if (TYPE_IMAGE == enterType) {
|
||||
linearParams.height = DisplayUtil.dip2px(this, 200);
|
||||
bottomLayout.setLayoutParams(linearParams);
|
||||
imageTrackingMobile = new ImageTrackingMobile(this);
|
||||
ret = imageTrackingMobile.loadModelFromBuf(IMAGE_SCENE_MS);
|
||||
} else if (TYPE_GARBAGE == enterType) {
|
||||
linearParams.height = DisplayUtil.dip2px(this, 100);
|
||||
bottomLayout.setLayoutParams(linearParams);
|
||||
garbageTrackingMobile = new GarbageTrackingMobile(this);
|
||||
String garbageModelPath = "model/garbage_mobilenetv2.ms";
|
||||
boolean garbageRet = garbageTrackingMobile.loadModelFromBuf(garbageModelPath);
|
||||
Log.d(TAG, "Garbage Loading model return value: " + garbageRet);
|
||||
ret = garbageTrackingMobile.loadModelFromBuf(GARBAGE_MS);
|
||||
} else if (TYPE_SCENE == enterType) {
|
||||
linearParams.height = DisplayUtil.dip2px(this, 100);
|
||||
bottomLayout.setLayoutParams(linearParams);
|
||||
sceneTrackingMobile = new SceneTrackingMobile(this);
|
||||
ret = sceneTrackingMobile.loadModelFromBuf(IMAGE_SCENE_MS);
|
||||
}
|
||||
Log.d(TAG, "Loading model return value: " + ret);
|
||||
cameraPreview.addImageRecognitionDataCallBack(this);
|
||||
}
|
||||
|
||||
|
|
@ -99,11 +114,7 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
@Override
|
||||
protected void onResume() {
|
||||
super.onResume();
|
||||
if (enterType == TYPE_DEMO) {
|
||||
cameraPreview.onResume(this, CameraPreview.OPEN_TYPE_IMAGE, mTrackingMobile);
|
||||
} else {
|
||||
cameraPreview.onResume(this, CameraPreview.OPEN_TYPE_IMAGE_CUSTOM, garbageTrackingMobile);
|
||||
}
|
||||
cameraPreview.onResume(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -115,20 +126,42 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
@Override
|
||||
protected void onStop() {
|
||||
super.onStop();
|
||||
if (mTrackingMobile != null) {
|
||||
boolean ret = mTrackingMobile.unloadModel();
|
||||
if (imageTrackingMobile != null) {
|
||||
boolean ret = imageTrackingMobile.unloadModel();
|
||||
Log.d(TAG, "Unload model return value: " + ret);
|
||||
}
|
||||
if (garbageTrackingMobile != null) {
|
||||
boolean ret = garbageTrackingMobile.unloadModel();
|
||||
Log.d(TAG, "garbageTrackingMobile Unload model return value: " + ret);
|
||||
}
|
||||
if (sceneTrackingMobile != null) {
|
||||
boolean ret = sceneTrackingMobile.unloadModel();
|
||||
Log.d(TAG, "garbageTrackingMobile Unload model return value: " + ret);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRecognitionBitmapCallBack(Bitmap bitmap) {
|
||||
String result = null;
|
||||
long startTime = System.currentTimeMillis();
|
||||
if (TYPE_IMAGE == enterType) {
|
||||
result = imageTrackingMobile.MindSpore_runnet(bitmap);
|
||||
} else if (TYPE_GARBAGE == enterType) {
|
||||
result = garbageTrackingMobile.MindSpore_runnet(bitmap);
|
||||
} else if (TYPE_SCENE == enterType) {
|
||||
result = sceneTrackingMobile.MindSpore_runnet(bitmap);
|
||||
}
|
||||
long endTime = System.currentTimeMillis();
|
||||
|
||||
onRecognitionDataCallBack(result, (endTime - startTime) + "ms ");
|
||||
if (!bitmap.isRecycled()) {
|
||||
bitmap.recycle();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void onRecognitionDataCallBack(final String result, final String time) {
|
||||
if (enterType == TYPE_DEMO) {
|
||||
if (TYPE_IMAGE == enterType) {
|
||||
if (recognitionObjectBeanList != null) {
|
||||
recognitionObjectBeanList.clear();
|
||||
} else {
|
||||
|
|
@ -152,22 +185,21 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
});
|
||||
}
|
||||
|
||||
runOnUiThread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
showResultsInBottomSheet(recognitionObjectBeanList, time);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
runOnUiThread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
showResultsInBottomSheetGarbage(result, time);
|
||||
}
|
||||
});
|
||||
runOnUiThread(() -> showResultsInBottomSheet(recognitionObjectBeanList, time));
|
||||
} else if (TYPE_GARBAGE == enterType) {
|
||||
runOnUiThread(() -> showResultsInBottomSheetGarbage(result, time));
|
||||
} else if (TYPE_SCENE == enterType) {
|
||||
if (!result.equals("") && result.contains(":")) {
|
||||
String[] resultArray = result.split(":");
|
||||
bean = new RecognitionImageBean(resultArray[0], Float.valueOf(resultArray[1]));
|
||||
}
|
||||
runOnUiThread(() -> showResultsInBottomSheetScene(bean, time));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
RecognitionImageBean bean;
|
||||
|
||||
@UiThread
|
||||
protected void showResultsInBottomSheet(List<RecognitionImageBean> list, String time) {
|
||||
bottomLayout.removeAllViews();
|
||||
|
|
@ -216,6 +248,32 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
}
|
||||
}
|
||||
|
||||
@UiThread
|
||||
protected void showResultsInBottomSheetScene(RecognitionImageBean recognitionObjectBean, String time) {
|
||||
bottomLayout.removeAllViews();
|
||||
if (recognitionObjectBean != null) {
|
||||
HorTextView horTextView = new HorTextView(this);
|
||||
horTextView.setLeftTitle(recognitionObjectBean.getName() + ":");
|
||||
horTextView.setRightContent(String.format("%.2f", (100 * recognitionObjectBean.getScore())) + "%");
|
||||
horTextView.setBottomLineVisible(View.VISIBLE);
|
||||
bottomLayout.addView(horTextView);
|
||||
|
||||
HorTextView horTimeView = new HorTextView(this);
|
||||
horTimeView.setLeftTitle("Inference Time：");
|
||||
horTimeView.setRightContent(time);
|
||||
horTimeView.setBottomLineVisible(View.INVISIBLE);
|
||||
bottomLayout.addView(horTimeView);
|
||||
} else {
|
||||
TextView textView = new TextView(this);
|
||||
textView.setLayoutParams(new LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT));
|
||||
textView.setText("Keep moving.");
|
||||
textView.setGravity(Gravity.CENTER);
|
||||
textView.setTextColor(Color.BLACK);
|
||||
textView.setTextSize(30);
|
||||
bottomLayout.addView(textView);
|
||||
}
|
||||
}
|
||||
|
||||
private void showLoadView() {
|
||||
TextView textView = new TextView(this);
|
||||
textView.setLayoutParams(new LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT));
|
||||
|
|
@ -225,4 +283,6 @@ public class ImageCameraActivity extends AppCompatActivity implements CameraPrev
|
|||
textView.setTextSize(30);
|
||||
bottomLayout.addView(textView);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,14 +20,12 @@ import android.content.res.AssetManager;
|
|||
import android.graphics.Bitmap;
|
||||
import android.util.Log;
|
||||
|
||||
import com.mindspore.imageobject.track.TrackListener;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashMap;
|
||||
|
||||
public class ObjectTrackingMobile implements TrackListener {
|
||||
public class ObjectTrackingMobile {
|
||||
private final static String TAG = "ObjectTrackingMobile";
|
||||
|
||||
static {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
*/
|
||||
package com.mindspore.imageobject.objectdetection.ui;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.os.Bundle;
|
||||
import android.text.TextUtils;
|
||||
import android.util.Log;
|
||||
|
|
@ -79,7 +80,7 @@ public class ObjectCameraActivity extends AppCompatActivity implements CameraPre
|
|||
@Override
|
||||
protected void onResume() {
|
||||
super.onResume();
|
||||
cameraPreview.onResume(this, CameraPreview.OPEN_TYPE_OBJECT, mTrackingMobile);
|
||||
cameraPreview.onResume(this);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -89,8 +90,7 @@ public class ObjectCameraActivity extends AppCompatActivity implements CameraPre
|
|||
cameraPreview.onPause();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRecognitionDataCallBack(String result, String time) {
|
||||
public void onRecognitionDataCallBack(String result) {
|
||||
if (TextUtils.isEmpty(result)) {
|
||||
mObjectRectView.clearCanvas();
|
||||
return;
|
||||
|
|
@ -99,4 +99,16 @@ public class ObjectCameraActivity extends AppCompatActivity implements CameraPre
|
|||
recognitionObjectBeanList = getRecognitionList(result);
|
||||
mObjectRectView.setInfo(recognitionObjectBeanList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRecognitionBitmapCallBack(Bitmap bitmap) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
String result = mTrackingMobile.MindSpore_runnet(bitmap);
|
||||
long endTime = System.currentTimeMillis();
|
||||
Log.d(TAG,"TrackingMobile inferenceTime:"+(endTime - startTime) + "ms ");
|
||||
onRecognitionDataCallBack(result);
|
||||
if (!bitmap.isRecycled()) {
|
||||
bitmap.recycle();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@ import androidx.appcompat.app.AppCompatActivity;
|
|||
import com.alibaba.android.arouter.facade.annotation.Route;
|
||||
import com.mindspore.imageobject.R;
|
||||
import com.mindspore.imageobject.objectdetection.bean.RecognitionObjectBean;
|
||||
import com.mindspore.imageobject.objectdetection.help.BitmapUtils;
|
||||
import com.mindspore.imageobject.objectdetection.help.DisplayUtil;
|
||||
import com.mindspore.imageobject.util.BitmapUtils;
|
||||
import com.mindspore.imageobject.util.DisplayUtil;
|
||||
import com.mindspore.imageobject.objectdetection.help.ObjectTrackingMobile;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import android.view.View;
|
|||
|
||||
import com.mindspore.imageobject.R;
|
||||
import com.mindspore.imageobject.objectdetection.bean.RecognitionObjectBean;
|
||||
import com.mindspore.imageobject.objectdetection.help.DisplayUtil;
|
||||
import com.mindspore.imageobject.util.DisplayUtil;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.mindspore.imageobject.objectdetection.help;
|
||||
package com.mindspore.imageobject.util;
|
||||
|
||||
import android.app.Activity;
|
||||
import android.database.Cursor;
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.mindspore.imageobject.objectdetection.help;
|
||||
package com.mindspore.imageobject.util;
|
||||
|
||||
import android.content.Context;
|
||||
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:app="http://schemas.android.com/apk/res-auto"
|
||||
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools"
|
||||
android:id="@+id/container"
|
||||
android:layout_width="match_parent"
|
||||
|
|
@ -11,19 +10,15 @@
|
|||
<com.mindspore.imageobject.camera.CameraPreview
|
||||
android:id="@+id/image_camera_preview"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintTop_toTopOf="parent" />
|
||||
android:layout_height="match_parent" />
|
||||
|
||||
|
||||
<androidx.appcompat.widget.Toolbar
|
||||
android:id="@+id/toolbar"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="?attr/actionBarSize"
|
||||
android:background="@color/mindspore_semi_transparent"
|
||||
app:layout_constraintTop_toTopOf="parent">
|
||||
android:layout_alignParentTop="true"
|
||||
android:background="@color/mindspore_semi_transparent">
|
||||
|
||||
<TextView
|
||||
android:layout_width="match_parent"
|
||||
|
|
@ -40,11 +35,11 @@
|
|||
<LinearLayout
|
||||
android:id="@+id/layout_bottom_content"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="200dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_alignParentBottom="true"
|
||||
android:background="@color/colorPrimary"
|
||||
android:gravity="center"
|
||||
android:orientation="vertical"
|
||||
app:layout_constraintBottom_toBottomOf="parent" />
|
||||
android:orientation="vertical" />
|
||||
|
||||
|
||||
</androidx.constraintlayout.widget.ConstraintLayout>
|
||||
</RelativeLayout>
|
||||
Loading…
Reference in New Issue