added handle for the empty return code file.

Author: yhwen

nvflare/private/fed/client/client_executor.py

@@ -427,7 +427,7 @@ def _wait_child_process_finish(self, client, job_id, allocated_resource, token,
         if child_process:
             child_process.wait()
 
-            return_code = get_return_code(child_process, job_id, workspace)
+            return_code = get_return_code(child_process, job_id, workspace, self.logger)
 
             self.logger.info(f"run ({job_id}): child worker process finished with RC {return_code}")
             if return_code in [ProcessExitCode.UNSAFE_COMPONENT, ProcessExitCode.CONFIG_ERROR]:

nvflare/private/fed/server/server_engine.py

@@ -211,7 +211,7 @@ def wait_for_complete(self, workspace, job_id, process):
                     break
                 time.sleep(0.1)
             with self.lock:
-                return_code = get_return_code(process, job_id, workspace)
+                return_code = get_return_code(process, job_id, workspace, self.logger)
                 # if process exit but with Execution exception
                 if return_code and return_code != 0:
                     self.logger.info(f"Job: {job_id} child process exit with return code {return_code}")

nvflare/private/fed/utils/fed_utils.py

@@ -317,16 +317,20 @@ def get_target_names(targets):
     return target_names
 
 
-def get_return_code(process, job_id, workspace):
+def get_return_code(process, job_id, workspace, logger):
     run_dir = os.path.join(workspace, job_id)
     rc_file = os.path.join(run_dir, FLMetaKey.PROCESS_RC_FILE)
-    try:
-        if os.path.exists(rc_file):
+    if os.path.exists(rc_file):
+        try:
             with open(rc_file, "r") as f:
                 return_code = int(f.readline())
             os.remove(rc_file)
-        else:
+        except Exception:
+            logger.warning(
+                f"Could not get the return_code from {rc_file} of the job:{job_id}, "
+                f"Return the RC from the process:{process.pid}"
+            )
             return_code = process.poll()
-        return return_code
-    except Exception:
-        raise RuntimeError(f"Could not get the return_code of the {job_id} execution, process_id:{process.pid}")
+    else:
+        return_code = process.poll()
+    return return_code