chore(backend): add new api in engine for monitor

backend/lib/restful_wrapper.py

@@ -21,3 +21,22 @@ def add_learnware_verified(learnware_id, check_status):
         raise Exception(f"Add learnware verified failed: {result['msg']}")
 
     return result
+
+
+def get_learnware(learnware_id):
+    host = context.config["backend_host"]
+    port = context.config["listen_port"]
+    url = f"http://{host}:{port}/engine/get_learnware"
+    data = {"learnware_id": learnware_id}
+
+    response = requests.get(url, params=data)
+
+    if response.status_code != 200:
+        raise Exception(f"Get learnware dirpath failed: {response.text}")
+
+    result = response.json()
+
+    if result["code"] != 0:
+        raise Exception(f"Get learnware dirpath failed: {result['msg']}")
+
+    return result["data"]["learnware_dirpath"], result["data"]["semantic_specification"]

backend/restful/engine.py

@@ -185,7 +185,7 @@ class DownloadLearnware(flask_restful.Resource):
 
         if verify_status == dbops.LearnwareVerifyStatus.SUCCESS.value:
             try:
-                learnware_zip_path = context.engine.get_learnware_path_by_ids(learnware_id)
+                learnware_zip_path = context.engine.get_learnware_zip_path_by_ids(learnware_id)
             except:
                 return {"code": 42, "msg": "Engine download learnware error."}, 200
 
@@ -243,7 +243,9 @@ class DownloadMultiLearnware(flask_restful.Resource):
             return {"code": 21, "msg": "Request parameters error."}, 200
 
         try:
-            learnware_paths = [context.engine.get_learnware_path_by_ids(learnware_id) for learnware_id in learnware_ids]
+            learnware_paths = [
+                context.engine.get_learnware_zip_path_by_ids(learnware_id) for learnware_id in learnware_ids
+            ]
         except:
             return {"code": 42, "msg": "Engine download learnware error."}, 200
 
@@ -262,8 +264,38 @@ class DownloadMultiLearnware(flask_restful.Resource):
         return res
 
 
+class GetLearnware(flask_restful.Resource):
+    def get(self):
+        learnware_id = request.args.get("learnware_id")
+
+        if learnware_id is None:
+            return {"code": 21, "msg": "Request parameters error."}, 200
+
+        try:
+            learnware = context.engine.get_learnware_by_ids(learnware_id)
+        except:
+            return {"code": 42, "msg": "Engine get learnware error."}, 200
+
+        if learnware is None:
+            learnware_dirpath = None
+            semantic_specification = None
+        else:
+            try:
+                learnware_dirpath = learnware.get_dirpath()
+                semantic_specification = learnware.get_specification().get_semantic_spec()
+            except:
+                return {"code": 42, "msg": "Engine get learnware path error."}, 200
+
+        return {
+            "code": 0,
+            "msg": "Ok",
+            "data": {"learnware_dirpath": learnware_dirpath, "semantic_specification": semantic_specification},
+        }, 200
+
+
 api.add_resource(SematicSpecification, "/semantic_specification")
 api.add_resource(SearchLearnware, "/search_learnware")
 api.add_resource(DownloadLearnware, "/download_learnware")
 api.add_resource(LearnwareInfo, "/learnware_info")
 api.add_resource(DownloadMultiLearnware, "/download_multi_learnware")
+api.add_resource(GetLearnware, "/get_learnware")

backend/scripts/monitor_learnware_verify.py

@@ -23,13 +23,11 @@ from learnware.learnware import get_learnware_from_dirpath
 
 
 def verify_learnware_with_conda_checker(
-    learnware_id: str, learnware_path: str, semantic_spec_filename: str
+    learnware_id: str, learnware_path: str, semantic_specification: dict
 ) -> Tuple[bool, str]:
     verify_sucess = True
     command_output = ""
     try:
-        with open(semantic_spec_filename, "r") as fin:
-            semantic_specification = json.load(fin)
         learnware = get_learnware_from_dirpath(
             id=learnware_id, semantic_spec=semantic_specification, learnware_dirpath=learnware_path, ignore_error=False
         )
@@ -86,30 +84,41 @@ def worker_process_func(q: queue.Queue, env: dict):
             continue
 
         dbops.update_learnware_verify_status(learnware_id, LearnwareVerifyStatus.PROCESSING)
-        learnware_filename = context.get_learnware_verify_file_path(learnware_id)
-        semantic_spec_filename = learnware_filename[:-4] + ".json"
-        process_result_filename = learnware_filename + ".result"
-        learnware_processed_filename = learnware_filename[:-4] + "_processed.zip"
-        learnware_check_status = BaseChecker.NONUSABLE_LEARNWARE
 
-        with tempfile.TemporaryDirectory(dir=context.config["temp_path"]) as tmpdir:
-            extract_path = tmpdir
-            if not os.path.exists(extract_path):
-                os.makedirs(extract_path)
-                pass
+        try:
+            learnware_dirpath, semantic_specification = restful_wrapper.get_learnware(learnware_id)
+            learnware_check_status = BaseChecker.NONUSABLE_LEARNWARE
 
-            context.logger.info(f"Extracting learnware to {extract_path}")
-            with zipfile.ZipFile(learnware_filename, "r") as zip_ref:
-                top_folder = common_utils.get_top_folder_in_zip(zip_ref)
-                zip_ref.extractall(extract_path)
-                pass
+            if learnware_dirpath is None:
+                learnware_filename = context.get_learnware_verify_file_path(learnware_id)
+                semantic_spec_filename = learnware_filename[:-4] + ".json"
+                process_result_filename = learnware_filename + ".result"
+                learnware_processed_filename = learnware_filename[:-4] + "_processed.zip"
 
-            extract_path = os.path.join(extract_path, top_folder)
-            update_learnware_yaml_file(extract_path, learnware_id, semantic_spec_filename)
+                with open(semantic_spec_filename, "r") as fin:
+                    semantic_specification = json.load(fin)
 
-            verify_success, command_output = verify_learnware_with_conda_checker(
-                learnware_id, extract_path, semantic_spec_filename
-            )
+                with tempfile.TemporaryDirectory(dir=context.config["temp_path"]) as tmpdir:
+                    extract_path = tmpdir
+                    if not os.path.exists(extract_path):
+                        os.makedirs(extract_path)
+
+                    context.logger.info(f"Extracting learnware to {extract_path}")
+                    with zipfile.ZipFile(learnware_filename, "r") as zip_ref:
+                        top_folder = common_utils.get_top_folder_in_zip(zip_ref)
+                        zip_ref.extractall(extract_path)
+
+                    extract_path = os.path.join(extract_path, top_folder)
+                    update_learnware_yaml_file(extract_path, learnware_id, semantic_spec_filename)
+
+                    verify_success, command_output = verify_learnware_with_conda_checker(
+                        learnware_id, extract_path, semantic_specification
+                    )
+                    learnware.utils.zip_learnware_folder(extract_path, learnware_processed_filename)
+            else:
+                verify_success, command_output = verify_learnware_with_conda_checker(
+                    learnware_id, learnware_dirpath, semantic_specification
+                )
 
             # the learnware my be deleted
             if not dbops.check_learnware_exist(learnware_id=learnware_id):
@@ -122,7 +131,10 @@ def worker_process_func(q: queue.Queue, env: dict):
             else:
                 verify_status = LearnwareVerifyStatus.FAIL
 
-            learnware.utils.zip_learnware_folder(extract_path, learnware_processed_filename)
+        except Exception as e:
+            print("-" * 100, str(e))
+            verify_status = LearnwareVerifyStatus.WAITING
+            command_output = "\n\n" + str(e)
 
         try:
             # internal service should not use proxy