hackathon

build.gradle

@@ -291,7 +291,7 @@ allprojects {
 
     // See please https://bugs.openjdk.java.net/browse/JDK-8209058
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
-      compile.options.compilerArgs << '-Werror'
+      //compile.options.compilerArgs << '-Werror'
     }
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'

buildSrc/build.gradle

@@ -252,7 +252,7 @@ if (project != rootProject) {
   // disable fail-on-warnings for this specific task which trips Java 11 bug
   // https://bugs.openjdk.java.net/browse/JDK-8209058
   tasks.named("compileTestFixturesJava").configure {
-    options.compilerArgs -= '-Werror'
+    //options.compilerArgs -= '-Werror'
   }
 
   tasks.register("integTest", Test) {

distribution/src/config/log4j2.properties

@@ -10,6 +10,21 @@
 #
 
 status = error
+log4j.logger.s3.stumpy=TRACE, Console;
+log4j.logger.s3.stumpy.Profiler=TRACE, Console;
+
+logger.s3.name=stumpy
+logger.s3.level = trace
+logger.s3.appenderRef.console.ref = console
+
+logger.s3.stumpy.Profiler.name=stumpy
+logger.s3.stumpy.Profiler.level = trace
+logger.s3.stumpy.Profiler.appenderRef.console.ref = console
+
+logger.s3.stumpy.name=stumpy
+logger.s3.stumpy.level = trace
+logger.s3.stumpy.appenderRef.console.ref = console
+
 
 appender.console.type = Console
 appender.console.name = console

server/build.gradle

@@ -69,8 +69,9 @@ dependencies {
   api project(":libs:opensearch-geo")
   api project(":libs:opensearch-telemetry")
   api project(":libs:opensearch-task-commons")
-
-
+  //api files('/Users/rajivkv/DFDDClient/src/DFDDClient/build/lib/DFDDClient-1.0.jar')
+  //api fileTree(dir: '/Users/rajivkv/DFDDClient/build/DFDDClient/DFDDClient-1.0/AL2_x86_64/DEV.STD.PTHREAD/build/lib/dependencies', include: ['*.jar'])
+    api fileTree(dir: '/Users/rajivkv/DFDDClient/build/DFDDClient/DFDDClient-1.2/AL2_x86_64/DEV.STD.PTHREAD/build/lib/dependencies', include: ['*.jar'])
   compileOnly project(':libs:opensearch-plugin-classloader')
   testRuntimeOnly project(':libs:opensearch-plugin-classloader')
 

server/src/main/java/org/opensearch/bootstrap/Bootstrap.java

@@ -221,23 +221,23 @@ public void run() {
             });
         }
 
-        try {
-            // look for jar hell
-            final Logger logger = LogManager.getLogger(JarHell.class);
-            JarHell.checkJarHell(logger::debug);
-        } catch (IOException | URISyntaxException e) {
-            throw new BootstrapException(e);
-        }
-
+//        try {
+//            // look for jar hell
+//            final Logger logger = LogManager.getLogger(JarHell.class);
+//            JarHell.checkJarHell(logger::debug);
+//        } catch (IOException | URISyntaxException e) {
+//            throw new BootstrapException(e);
+//        }
+//
         // Log ifconfig output before SecurityManager is installed
         IfConfig.logIfNecessary();
 
         // install SM after natives, shutdown hooks, etc.
-        try {
-            Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
-        } catch (IOException | NoSuchAlgorithmException e) {
-            throw new BootstrapException(e);
-        }
+//        try {
+//            Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
+//        } catch (IOException | NoSuchAlgorithmException e) {
+//            throw new BootstrapException(e);
+//        }
 
         node = new Node(environment) {
             @Override

server/src/main/java/org/opensearch/bootstrap/OpenSearch.java

@@ -52,6 +52,8 @@
 import java.security.Security;
 import java.util.Arrays;
 import java.util.Locale;
+import s3.dfddclient.DFDDClient;
+import s3.stumpy.StumpyManager;
 
 /**
  * This class starts opensearch.
@@ -80,11 +82,13 @@ class OpenSearch extends EnvironmentAwareCommand {
             .availableUnless(daemonizeOption);
     }
 
+
     /**
      * Main entry point for starting opensearch
      */
     @SuppressWarnings("removal")
     public static void main(final String[] args) throws Exception {
+
         overrideDnsCachePolicyProperties();
         /*
          * We want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the
@@ -99,6 +103,7 @@ public void checkPermission(Permission perm) {
             }
 
         });
+
         LogConfigurator.registerErrorListener();
         final OpenSearch opensearch = new OpenSearch();
         int status = main(args, opensearch, Terminal.DEFAULT);

server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java

@@ -49,6 +49,7 @@
 import org.opensearch.core.transport.TransportResponse.Empty;
 import org.opensearch.monitor.NodeHealthService;
 import org.opensearch.monitor.StatusInfo;
+import org.opensearch.node.Node;
 import org.opensearch.threadpool.ThreadPool.Names;
 import org.opensearch.transport.ConnectTransportException;
 import org.opensearch.transport.Transport;
@@ -70,6 +71,8 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
+import s3.dfddclient.DFDDClient;
+
 import static org.opensearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.opensearch.monitor.StatusInfo.Status.UNHEALTHY;
 
@@ -131,6 +134,8 @@ public class FollowersChecker {
     private volatile FastResponseState fastResponseState;
     private ClusterManagerMetrics clusterManagerMetrics;
 
+    private boolean followerCheckerDisabled = false;
+
     public FollowersChecker(
         Settings settings,
         ClusterSettings clusterSettings,
@@ -163,10 +168,16 @@ public FollowersChecker(
         transportService.addConnectionListener(new TransportConnectionListener() {
             @Override
             public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) {
-                handleDisconnectedNode(node);
+                if(followerCheckerDisabled) {
+                    handleDisconnectedNode(node);
+                }
             }
         });
         this.clusterManagerMetrics = clusterManagerMetrics;
+        followerCheckerDisabled = settings.getAsBoolean(Node.NODE_ATTRIBUTES.getKey() + "follower_check.disabled", false);
+        if (followerCheckerDisabled) {
+            logger.info("Follower check is disabled");
+        }
     }
 
     private void setFollowerCheckInterval(TimeValue followerCheckInterval) {
@@ -237,6 +248,7 @@ private void handleFollowerCheck(FollowerCheckRequest request, TransportChannel
         if (request.term < responder.term) {
             throw new CoordinationStateRejectedException("rejecting " + request + " since local state is " + this);
         }
+        DFDDClient dfddClient;
 
         transportService.getThreadPool().generic().execute(new AbstractRunnable() {
             @Override
@@ -476,7 +488,7 @@ public String toString() {
     }
 
     /**
-     * Request to check follower.
+     * Request to check follower:.
      *
      * @opensearch.internal
      */

server/src/main/java/org/opensearch/common/logging/LogConfigurator.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.common.logging;
 
+import java.io.File;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
@@ -177,11 +178,16 @@ private static void configure(final Settings settings, final Path configsPath, f
             @Override
             public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                 if (file.getFileName().toString().equals("log4j2.properties")) {
-                    configurations.add((PropertiesConfiguration) factory.getConfiguration(context, file.toString(), file.toUri()));
+                    System.out.println("config file" + file.toFile().getAbsoluteFile());
+
+                 //   configurations.add((PropertiesConfiguration) factory.getConfiguration(context, file.toString(), file.toUri()));
                 }
                 return FileVisitResult.CONTINUE;
             }
         });
+        File fi = new File("/Users/rajivkv/opensearch-main/log4j2.properties");
+        configurations.add((PropertiesConfiguration) factory.getConfiguration(context, fi.toString(), fi.toURI()));
+
 
         if (configurations.isEmpty()) {
             throw new UserException(ExitCodes.CONFIG, "no log4j2.properties found; tried [" + configsPath + "] and its subdirectories");
@@ -199,7 +205,7 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
 
     private static void configureStatusLogger() {
         final ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
-        builder.setStatusLevel(Level.ERROR);
+        builder.setStatusLevel(Level.DEBUG);
         Configurator.initialize(builder.build());
     }
 
@@ -220,6 +226,7 @@ private static void configureLoggerLevels(final Settings settings) {
                 final Level level = s.get(settings);
                 Loggers.setLevel(LogManager.getLogger(s.getKey().substring("logger.".length())), level);
             });
+
     }
 
     /**

server/src/main/java/org/opensearch/common/logging/Loggers.java

@@ -59,7 +59,7 @@ public class Loggers {
 
     public static final Setting<Level> LOG_DEFAULT_LEVEL_SETTING = new Setting<>(
         "logger.level",
-        Level.INFO.name(),
+        Level.DEBUG.name(),
         Level::valueOf,
         Setting.Property.NodeScope
     );

server/src/main/java/org/opensearch/node/Node.java

@@ -278,6 +278,7 @@
 import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService;
 import org.opensearch.wlm.listeners.QueryGroupRequestOperationListener;
 import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService;
+import s3.dfddclient.DFDDClient;
 
 import javax.net.ssl.SNIHostName;
 
@@ -310,6 +311,8 @@
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import s3.dfddclient.HealthChecker;
+import s3.stumpy.StumpyManager;
 
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.common.util.FeatureFlags.BACKGROUND_TASK_EXECUTION_EXPERIMENTAL;
@@ -1776,11 +1779,42 @@ public void onTimeout(TimeValue timeout) {
 
         logger.info("started");
 
+
+
         pluginsService.filterPlugins(ClusterPlugin.class).forEach(plugin -> plugin.onNodeStarted(clusterService.localNode()));
+        registerToDFDD(clusterService);
 
         return this;
     }
 
+    private void registerToDFDD(ClusterService clusterService) {
+        logger.info("Registering with DFDD for health checks");
+        try {
+            String privateIp= clusterService.getSettings().get(Node.NODE_ATTRIBUTES.getKey() + "private_ip", "localhost");
+            StumpyManager mgr = StumpyManager.newOneThreadManager(StumpyManager.AssertionCheckFlag.DO_NOT_CHECK_THAT_ASSERTIONS_ARE_ENABLED);
+            DFDDClient dfddClient = new DFDDClient(mgr, "localhost", 2977 , "OzoneDFDDRuntimeClique-Desktop");
+            logger.info("Adding application {} DFDD for health checks on node {}" , clusterService.getClusterName().value(), privateIp);
+            dfddClient.addBeatingApplication(clusterService.getClusterName().value(), privateIp,
+                clusterService.getNodeName() + ":9200", new HealthChecker() {
+                    @Override
+                    public boolean isHealthy() {
+                        logger.info("DFDD health check");
+                        return true;
+                    }
+
+                    @Override
+                    public void doneChecking() {
+
+                    }
+                }
+            );
+
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException("stumpy", e);
+        }
+    }
+
     protected void configureNodeAndClusterIdStateListener(ClusterService clusterService) {
         NodeAndClusterIdStateListener.getAndSetNodeIdAndClusterId(
             clusterService,

server/src/main/java/org/opensearch/plugins/PluginsService.java

@@ -540,7 +540,7 @@ private List<Tuple<PluginInfo, Plugin>> loadBundles(Set<Bundle> bundles) {
         Map<String, Set<URL>> transitiveUrls = new HashMap<>();
         List<Bundle> sortedBundles = sortBundles(bundles);
         for (Bundle bundle : sortedBundles) {
-            checkBundleJarHell(JarHell.parseClassPath(), bundle, transitiveUrls);
+            //checkBundleJarHell(JarHell.parseClassPath(), bundle, transitiveUrls);
 
             final Plugin plugin = loadBundle(bundle, loaded);
             plugins.add(new Tuple<>(bundle.plugin, plugin));

server/src/main/resources/org/opensearch/bootstrap/security.policy

@@ -101,6 +101,7 @@ grant {
   // checked by scripting engines, and before hacks and other issues in
   // third party code, to safeguard these against unprivileged code like scripts.
   permission org.opensearch.SpecialPermission;
+  permission java.lang.RuntimePermission "*";
 
   // Allow host/ip name service lookups
   permission java.net.SocketPermission "*", "resolve";