Merge branch 'master' into endpoint-exclude

Author: omursahin

.github/workflows/ci.yml

@@ -117,9 +117,9 @@ jobs:
       # Make test report accessible from GitHub Actions (as Maven logs are long)
       - name: Publish Test Report
         if: success() || failure()
-        uses: mikepenz/action-junit-report@v4
+        uses: mikepenz/action-junit-report@v5
         env:
-          NODE_OPTIONS: "--max_old_space_size=6144"
+          NODE_OPTIONS: "--max_old_space_size=8000"
         with:
           report_paths: '**/target/surefire-reports/TEST-*.xml'
       # Upload coverage results

README.md

@@ -34,16 +34,48 @@ building on decades of research in the field of [Search-Based Software Testing](
 ### 1-Minute Example 
 
 On a console, copy&paste the following (requires _Docker_ installed).
-It will fuzz the PetClinic example API from Swagger, for 30 seconds. 
+It will fuzz the PetClinic example API from Swagger, for 30 seconds, as shown in the following video. 
 
 ```
 docker run -v "$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster  --blackBox true --maxTime 30s  --ratePerMinute 60 --bbSwaggerUrl  https://petstore.swagger.io/v2/swagger.json
 ```
-Note, if run in a MSYS shell on Windows like _Git Bash_, there is the need of an extra / before the $ (as in the following video).
 
 
 ![](docs/img/evomaster_docker_use.gif)
 
+#### Using Docker on Different Shells
+
+Note that, depending on which shell and operating system you are using, you might need slightly different commands when mounting folders with the `-v` option.  
+
+For example, if run in a MSYS shell on Windows like _Git Bash_, there is the need of an extra / before the $.
+
+```
+docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster  --blackBox true --maxTime 30s  --ratePerMinute 60 --bbSwaggerUrl  https://petstore.swagger.io/v2/swagger.json
+```
+
+If you are rather using a Command Prompt (Cmd.exe) terminal, you need to use `%CD%` instead of `$(pwd)` to refer to the current folder:
+
+```
+docker run -v %CD%/generated_tests:/generated_tests webfuzzing/evomaster  --blackBox true --maxTime 30s  --ratePerMinute 60 --bbSwaggerUrl  https://petstore.swagger.io/v2/swagger.json
+```
+
+On the other hand, on a PowerShell you need `${PWD}`:
+
+```
+docker run -v ${PWD}/generated_tests:/generated_tests webfuzzing/evomaster  --blackBox true --maxTime 30s  --ratePerMinute 60 --bbSwaggerUrl  https://petstore.swagger.io/v2/swagger.json
+```
+
+#### Troubleshooting
+
+If you encounter issues running the command:
+
+* Ensure Docker is installed and running.
+* Check that you have the correct rights/permissions to mount the specified volume.
+* Consult the Docker documentation for your shell environment for specific syntax requirements.
+
+
+#### Generated Output
+
 Once the command is executed, you can inspect the generated files under `generated_tests` folder. 
 
 Note, since version 4.0.0, now _EvoMaster_ by default also creates an interactive web report.

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java

@@ -9,9 +9,9 @@ public class ExtraHeuristicEntryDto implements Serializable {
 
     /**
      * The type of extra heuristic.
-     * Note: for the moment, we only have heuristics on SQL commands
+     * Note: for the moment, we only have heuristics on SQL, MONGO and OPENSEARCH commands
      */
-    public enum Type {SQL, MONGO}
+    public enum Type {SQL, MONGO, OPENSEARCH}
 
     /**
      * Should we try to minimize or maximize the heuristic?

client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java

@@ -179,6 +179,7 @@ default void extractRPCSchema(){}
 
     default Object getMongoConnection() {return null;}
 
+    default Object getOpenSearchConnection() {return null;}
 
     /**
      * <p>

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java

@@ -387,6 +387,7 @@ public Response runSut(SutRunDto dto, @Context HttpServletRequest httpServletReq
                         noKillSwitch(() -> sutController.initSqlHandler());
                         noKillSwitch(() -> sutController.registerOrExecuteInitSqlCommandsIfNeeded(true));
                         noKillSwitch(() -> sutController.initMongoHandler());
+                        noKillSwitch(() -> sutController.initOpenSearchHandler());
                     } else {
                         //TODO as starting should be blocking, need to check
                         //if initialized, and wait if not

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java

@@ -27,6 +27,7 @@
 import org.evomaster.client.java.controller.api.dto.problem.RPCProblemDto;
 import org.evomaster.client.java.controller.api.dto.problem.rpc.*;
 import org.evomaster.client.java.controller.api.dto.problem.rpc.RPCTestDto;
+import org.evomaster.client.java.controller.internal.db.OpenSearchHandler;
 import org.evomaster.client.java.sql.DbCleaner;
 import org.evomaster.client.java.sql.SqlScriptRunner;
 import org.evomaster.client.java.sql.SqlScriptRunnerCached;
@@ -86,6 +87,8 @@ public abstract class SutController implements SutHandler, CustomizationHandler
 
     private final MongoHandler mongoHandler = new MongoHandler();
 
+    private final OpenSearchHandler openSearchHandler = new OpenSearchHandler();
+
     private Server controllerServer;
 
     /**
@@ -323,6 +326,18 @@ public final void initMongoHandler() {
         }
     }
 
+    // TODO: Refactor this initialization methods once Redis and OpenSearch implementations are done
+    public final void initOpenSearchHandler() {
+        // This is needed because the replacement use to get this info occurs during the start of the SUT.
+        Object connection = getOpenSearchConnection();
+        openSearchHandler.setOpenSearchClient(connection);
+
+        List<AdditionalInfo> list = getAdditionalInfoList();
+        if (!list.isEmpty()) {
+            AdditionalInfo last = list.get(list.size() - 1);
+            last.getOpenSearchInfoData().forEach(openSearchHandler::handle);
+        }
+    }
 
     /**
      * TODO further handle multiple connections
@@ -366,7 +381,7 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase
 
         ExtraHeuristicsDto dto = new ExtraHeuristicsDto();
 
-        if (isSQLHeuristicsComputationAllowed() || isMongoHeuristicsComputationAllowed()) {
+        if (isSQLHeuristicsComputationAllowed() || isMongoHeuristicsComputationAllowed() || isOpenSearchHeuristicsComputationAllowed()) {
             List<AdditionalInfo> additionalInfoList = getAdditionalInfoList();
 
             if (isSQLHeuristicsComputationAllowed()) {
@@ -375,6 +390,10 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase
             if (isMongoHeuristicsComputationAllowed()) {
                 computeMongoHeuristics(dto, additionalInfoList);
             }
+
+            if (isOpenSearchHeuristicsComputationAllowed()) {
+                computeOpenSearchHeuristics(dto, additionalInfoList);
+            }
         }
         return dto;
     }
@@ -387,6 +406,10 @@ private boolean isMongoHeuristicsComputationAllowed() {
         return mongoHandler.isCalculateHeuristics() || mongoHandler.isExtractMongoExecution();
     }
 
+    private boolean isOpenSearchHeuristicsComputationAllowed() {
+        return openSearchHandler.isCalculateHeuristics();
+    }
+
     private void computeSQLHeuristics(ExtraHeuristicsDto dto, List<AdditionalInfo> additionalInfoList, boolean queryFromDatabase) {
         /*
         TODO refactor, once we move SQL analysis into Core
@@ -467,6 +490,34 @@ public final void computeMongoHeuristics(ExtraHeuristicsDto dto, List<Additional
         }
     }
 
+    public final void computeOpenSearchHeuristics(ExtraHeuristicsDto dto, List<AdditionalInfo> additionalInfoList) {
+        if (openSearchHandler.isCalculateHeuristics()) {
+            if (!additionalInfoList.isEmpty()) {
+                AdditionalInfo last = additionalInfoList.get(additionalInfoList.size() - 1);
+                last.getOpenSearchInfoData().forEach(it -> {
+                    try {
+                        openSearchHandler.handle(it);
+                    } catch (Exception e){
+                        SimpleLogger.error("FAILED TO HANDLE OPENSEARCH COMMAND", e);
+                        assert false;
+                    }
+                });
+            }
+
+            openSearchHandler.getEvaluatedOpenSearchCommands().stream()
+                .map(p ->
+                    new ExtraHeuristicEntryDto(
+                        ExtraHeuristicEntryDto.Type.OPENSEARCH,
+                        ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO,
+                        p.getCommand().toString(),
+                        p.getDistanceWithMetrics().getDistance(),
+                        p.getDistanceWithMetrics().getNumberOfEvaluatedDocuments(),
+                        false
+                    ))
+                .forEach(h -> dto.heuristics.add(h));
+        }
+
+    }
 
     /**
      * handle specified init sql script after SUT is started.

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/OpenSearchCommandWithDistance.java

@@ -0,0 +1,20 @@
+package org.evomaster.client.java.controller.internal.db;
+
+public class OpenSearchCommandWithDistance {
+    private final Object command;
+
+    private final OpenSearchDistanceWithMetrics distanceWithMetrics;
+
+    public OpenSearchCommandWithDistance(Object command, OpenSearchDistanceWithMetrics distanceWithMetrics) {
+        this.command = command;
+        this.distanceWithMetrics = distanceWithMetrics;
+    }
+
+    public Object getCommand() {
+        return command;
+    }
+
+    public OpenSearchDistanceWithMetrics getDistanceWithMetrics() {
+        return distanceWithMetrics;
+    }
+}

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/OpenSearchDistanceWithMetrics.java

@@ -0,0 +1,29 @@
+package org.evomaster.client.java.controller.internal.db;
+
+public class OpenSearchDistanceWithMetrics {
+
+    private final double distance;
+
+    private final int numberOfEvaluatedDocuments;
+
+    public OpenSearchDistanceWithMetrics(double distance, int numberOfEvaluatedDocuments) {
+        if (distance < 0) {
+            throw new IllegalArgumentException("distance must be non-negative but value is " + distance);
+        }
+
+        if (numberOfEvaluatedDocuments < 0) {
+            throw new IllegalArgumentException("numberOfEvaluatedDocuments must be non-negative but value is " + numberOfEvaluatedDocuments);
+        }
+
+        this.distance = distance;
+        this.numberOfEvaluatedDocuments = numberOfEvaluatedDocuments;
+    }
+
+    public double getDistance() {
+        return distance;
+    }
+
+    public int getNumberOfEvaluatedDocuments() {
+        return numberOfEvaluatedDocuments;
+    }
+}

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/OpenSearchHandler.java

@@ -0,0 +1,143 @@
+package org.evomaster.client.java.controller.internal.db;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import org.evomaster.client.java.controller.opensearch.OpenSearchHeuristicsCalculator;
+import org.evomaster.client.java.instrumentation.OpenSearchCommand;
+import org.evomaster.client.java.utils.SimpleLogger;
+
+public class OpenSearchHandler {
+
+    public static final String OPENSEARCH_CLIENT_CLASS_NAME = "org.opensearch.client.opensearch.OpenSearchClient";
+    public static final String OPENSEARCH_CLIENT_INDEX_METHOD_NAME = "index";
+    public static final String OPENSEARCH_CLIENT_SIZE_METHOD_NAME = "size";
+    public static final String OPENSEARCH_CLIENT_BUILD_METHOD_NAME = "build";
+    public static final String OPENSEARCH_CLIENT_SEARCH_METHOD_NAME = "search";
+    public static final String OPENSEARCH_CLIENT_HITS_METHOD_NAME = "hits";
+
+    public static final String OPENSEARCH_CLIENT_SEARCH_REQUEST_CLASS_NAME = "org.opensearch.client.opensearch.core.SearchRequest";
+    public static final String OPENSEARCH_CLIENT_SEARCH_REQUEST_BUILDER_CLASS_NAME = "org.opensearch.client.opensearch.core.SearchRequest$Builder";
+
+    private final List<OpenSearchCommand> commands;
+
+    private final boolean calculateHeuristics;
+
+    /**
+     * The heuristics based on the OpenSearch execution
+     */
+    private final List<OpenSearchCommandWithDistance> commandsWithDistances;
+
+    private final OpenSearchHeuristicsCalculator calculator = new OpenSearchHeuristicsCalculator();
+
+    private Object openSearchClient = null;
+
+    public OpenSearchHandler() {
+        this.commands = new ArrayList<>();
+        this.commandsWithDistances = new ArrayList<>();
+        this.calculateHeuristics = true;
+    }
+
+    public boolean isCalculateHeuristics() {
+        return calculateHeuristics;
+    }
+
+    public void handle(OpenSearchCommand command) {
+        commands.add(command);
+    }
+
+    public List<OpenSearchCommandWithDistance> getEvaluatedOpenSearchCommands() {
+        commands.stream()
+            .filter(command -> command.getQuery() != null)
+            .forEach(openSearchCommand -> {
+                OpenSearchDistanceWithMetrics distanceWithMetrics = computeCommandDistance(openSearchCommand);
+                commandsWithDistances.add(new OpenSearchCommandWithDistance(openSearchCommand, distanceWithMetrics));
+        });
+
+        commands.clear();
+
+        return commandsWithDistances;
+    }
+
+    private OpenSearchDistanceWithMetrics computeCommandDistance(OpenSearchCommand command) {
+        List<String> indexName = command.getIndex();
+        List<?> documents = getDocuments(indexName);
+
+        double min = Double.MAX_VALUE;
+        int numberOfEvaluatedDocuments = 0;
+        for (Object doc : documents) {
+            numberOfEvaluatedDocuments += 1;
+            double findDistance;
+            try {
+                findDistance = calculator.computeExpression(command.getQuery(), doc);
+            } catch (Exception ex) {
+                SimpleLogger.uniqueWarn("Failed to compute find: " + command.getQuery() + " with data " + doc);
+                findDistance = Double.MAX_VALUE;
+            }
+
+            if (findDistance == 0) {
+                return new OpenSearchDistanceWithMetrics(0, numberOfEvaluatedDocuments);
+            } else if (findDistance < min) {
+                min = findDistance;
+            }
+        }
+
+        return new OpenSearchDistanceWithMetrics(min, numberOfEvaluatedDocuments);
+    }
+
+    private List<Object> getDocuments(List<String> indexNames) {
+        try {
+            // Get the OpenSearchClient class
+            Class<?> openSearchClientClass = openSearchClient.getClass();
+
+            // Get SearchRequest.Builder class
+            Class<?> searchRequestBuilderClass = Class.forName(OPENSEARCH_CLIENT_SEARCH_REQUEST_BUILDER_CLASS_NAME);
+            Object searchRequestBuilder = searchRequestBuilderClass.getDeclaredConstructor().newInstance();
+
+            // Set the index/indices on the builder
+            Method indexMethod = searchRequestBuilderClass.getMethod(OPENSEARCH_CLIENT_INDEX_METHOD_NAME, List.class);
+            indexMethod.invoke(searchRequestBuilder, indexNames);
+
+            // Set a large size to get all documents (could use scroll for large datasets)
+            Method sizeMethod = searchRequestBuilderClass.getMethod(OPENSEARCH_CLIENT_SIZE_METHOD_NAME, Integer.class);
+            sizeMethod.invoke(searchRequestBuilder, 10000); // Max 10k documents
+
+            // Build the SearchRequest
+            Method buildMethod = searchRequestBuilderClass.getMethod(OPENSEARCH_CLIENT_BUILD_METHOD_NAME);
+            Object searchRequest = buildMethod.invoke(searchRequestBuilder);
+
+            // Execute the search: openSearchClient.search(searchRequest, Object.class)
+            Method searchMethod = openSearchClientClass.getMethod(OPENSEARCH_CLIENT_SEARCH_METHOD_NAME,
+                Class.forName(OPENSEARCH_CLIENT_SEARCH_REQUEST_CLASS_NAME),
+                Class.class);
+            Object searchResponse = searchMethod.invoke(openSearchClient, searchRequest, Object.class);
+
+            Method hitsMethod = searchResponse.getClass().getMethod(OPENSEARCH_CLIENT_HITS_METHOD_NAME);
+            Object hitsContainer = hitsMethod.invoke(searchResponse);
+
+            Method hitsListMethod = hitsContainer.getClass().getMethod(OPENSEARCH_CLIENT_HITS_METHOD_NAME);
+            Object hitsList = hitsListMethod.invoke(hitsContainer);
+
+            // loop hitslist as if it was a list
+            List<Object> result = new ArrayList<>();
+            for (Object hit : (List<Object>)hitsList) {
+                Method hitSourceMethod = hit.getClass().getMethod("source");
+                result.add(hitSourceMethod.invoke(hit));
+            }
+
+            return result; // Returns List<Document>
+
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to retrieve documents from OpenSearch indices: " + indexNames, e);
+        }
+    }
+
+    private Object getDocuments() {
+        // Default implementation - this method should be called with specific index names
+        return null;
+    }
+
+    public void setOpenSearchClient(Object openSearchClient) {
+        this.openSearchClient = openSearchClient;
+    }
+}