fix: force shutdown unclosed services (#1743)

(cherry picked from commit 4ab2587)

Author: limingyaoo

src/main/java/com/aws/greengrass/lifecyclemanager/GenericExternalService.java

@@ -600,6 +600,23 @@ protected void shutdown() {
         }
     }
 
+    /**
+     * Shutdown a service without running the shutdown script.
+     * Including stop processes and clean up resources.
+     */
+    public void forceShutdown() {
+
+        stopAllLifecycleProcessesWithoutLock(lifecycleProcesses);
+
+        // Clean up any resource manager entities (can be OS specific) that might have been created for this
+        // component.
+        systemResourceController.removeResourceController(this);
+
+        logger.atInfo().setEventType("force-service-shutdown").log();
+
+        resetRunWith();
+    }
+
     /**
      * Stop all the lifecycle processes.
      *
@@ -631,6 +648,27 @@ private void stopProcesses(List<Exec> processes) {
         }
     }
 
+    /**
+     * Force stop processes without acquiring the lock.
+     */
+    @SuppressWarnings("PMD.CloseResource")
+    private void stopAllLifecycleProcessesWithoutLock(List<Exec> processes) {
+        for (Exec e : processes) {
+            if (e != null && e.isRunning()) {
+                logger.atInfo().log("Force shutting down process {}", e);
+                try {
+                    e.close();
+                    logger.atInfo().log("Force shutdown completed for process {}", e);
+                    processes.remove(e);
+                } catch (IOException ex) {
+                    logger.atWarn().log("Force shutdown timed out for process {}", e);
+                }
+            } else {
+                processes.remove(e);
+            }
+        }
+    }
+
     @Override
     public void handleError() throws InterruptedException {
         if (getConfig().findNode(GreengrassService.SERVICE_LIFECYCLE_NAMESPACE_TOPIC,

src/main/java/com/aws/greengrass/lifecyclemanager/KernelLifecycle.java

@@ -504,7 +504,23 @@ public void stopAllServices(int timeoutSeconds) {
             List<String> unclosedServices =
                     IntStream.range(0, arr.length).filter((i) -> !arr[i].isDone() || arr[i].isCompletedExceptionally())
                             .mapToObj((i) -> d[i].getName()).collect(Collectors.toList());
-            logger.atError("services-shutdown-errored", e).kv("unclosedServices", unclosedServices).log();
+
+            logger.atError("services-shutdown-errored", e).kv("unclosedServices", unclosedServices)
+                    .log("Force closing services");
+
+            for (String unclosedService : unclosedServices) {
+                try {
+                    GreengrassService service = kernel.locate(unclosedService);
+                    // for generic external service, kill processes and clean up
+                    if (service instanceof GenericExternalService) {
+                        GenericExternalService genericService = (GenericExternalService) service;
+                        genericService.forceShutdown();
+                    }
+                } catch (ServiceLoadException ex) {
+                    logger.atWarn().log("Could not locate service: {} for forced process cleanup. "
+                            + "Skipping cleanup for this service.", unclosedService, ex);
+                }
+            }
         }
     }