diff --git a/model_zoo/wide_and_deep/metrics.py b/model_zoo/wide_and_deep/metrics.py new file mode 100644 index 00000000000..277d6744dc9 --- /dev/null +++ b/model_zoo/wide_and_deep/metrics.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Area under cure metric +""" + +from mindspore.nn.metrics import Metric +from sklearn.metrics import roc_auc_score + +class AUCMetric(Metric): + """ + Area under cure metric + """ + + def __init__(self): + super(AUCMetric, self).__init__() + self.clear() + + def clear(self): + """Clear the internal evaluation result.""" + self.true_labels = [] + self.pred_probs = [] + + def update(self, *inputs): # inputs + all_predict = inputs[1].asnumpy() # predict + all_label = inputs[2].asnumpy() # label + self.true_labels.extend(all_label.flatten().tolist()) + self.pred_probs.extend(all_predict.flatten().tolist()) + + def eval(self): + if len(self.true_labels) != len(self.pred_probs): + raise RuntimeError( + 'true_labels.size is not equal to pred_probs.size()') + + auc = roc_auc_score(self.true_labels, self.pred_probs) + print("====" * 20 + " auc_metric end") + print("====" * 20 + " auc: {}".format(auc)) + return auc diff --git a/model_zoo/wide_and_deep/test.py b/model_zoo/wide_and_deep/test.py new file mode 100644 index 00000000000..a8eb0cdbc47 --- /dev/null +++ b/model_zoo/wide_and_deep/test.py @@ -0,0 +1,94 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" test_training """ + +import os + +from mindspore import Model, context +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from wide_deep.models.WideDeep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel +from wide_deep.utils.callbacks import LossCallBack, EvalCallBack +from wide_deep.data.datasets import create_dataset +from wide_deep.utils.metrics import AUCMetric +from tools.config import Config_WideDeep + +context.set_context(mode=context.GRAPH_MODE, device_target="Davinci", + save_graphs=True) + + +def get_WideDeep_net(config): + WideDeep_net = WideDeepModel(config) + + loss_net = NetWithLossClass(WideDeep_net, config) + train_net = TrainStepWrap(loss_net) + eval_net = PredictWithSigmoid(WideDeep_net) + + return train_net, eval_net + + +class ModelBuilder(): + """ + Wide and deep model builder + """ + def __init__(self): + pass + + def get_hook(self): + pass + + def get_train_hook(self): + hooks = [] + callback = LossCallBack() + hooks.append(callback) + + if int(os.getenv('DEVICE_ID')) == 0: + pass + return hooks + + def get_net(self, config): + return get_WideDeep_net(config) + + +def test_eval(config): + """ + test evaluate + """ + data_path = config.data_path + batch_size = config.batch_size + ds_eval = create_dataset(data_path, train_mode=False, epochs=2, + batch_size=batch_size) + print("ds_eval.size: {}".format(ds_eval.get_dataset_size())) + + net_builder = ModelBuilder() + train_net, eval_net = net_builder.get_net(config) + + param_dict = load_checkpoint(config.ckpt_path) + load_param_into_net(eval_net, param_dict) + + auc_metric = AUCMetric() + model = Model(train_net, eval_network=eval_net, metrics={"auc": auc_metric}) + + eval_callback = EvalCallBack(model, ds_eval, auc_metric, config) + + model.eval(ds_eval, callbacks=eval_callback) + + +if __name__ == "__main__": + widedeep_config = Config_WideDeep() + widedeep_config.argparse_init() + + test_eval(widedeep_config.widedeep)