Setup Cassandra benchmark env and add CQLMultiQueryBenchmark

Signed-off-by: Boxuan Li <[email protected]>

Author: li-boxuan

.github/workflows/ci-benchmark.yml

@@ -44,11 +44,18 @@ jobs:
         with:
           java-version: 1.8
       - run: mvn clean install --projects janusgraph-all -Pjanusgraph-cache -Dmaven.javadoc.skip=true ${{ env.BUILD_MAVEN_OPTS }}
-
+      - run: mvn verify --projects janusgraph-all -Pjanusgraph-cache ${{ env.VERIFY_MAVEN_OPTS }}
   benchmark:
     name: Performance regression check
     runs-on: ubuntu-latest
+    needs: build-all
     steps:
+      # install CCM for CQL benchmarks
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - run: pip install git+https://github.com/li-boxuan/ccm.git
+
       - uses: actions/checkout@v2
       - uses: actions/cache@v2
         with:
@@ -61,6 +68,9 @@ jobs:
           java-version: 1.8
       - run: mvn clean install -Pjanusgraph-benchmark ${{ env.BUILD_MAVEN_OPTS }} -Dgpg.skip=true
       - run: mvn verify --projects janusgraph-benchmark
+        env:
+          CCM_CLUSTER_START_DEFAULT_TIMEOUT: 300
+          CCM_UPDATE_PID_DEFAULT_TIMEOUT: 300
 
       # Download previous benchmark result from cache (if exists)
       - name: Download previous benchmark data

janusgraph-benchmark/pom.xml

@@ -19,6 +19,12 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.janusgraph</groupId>
+            <artifactId>janusgraph-cql</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <!-- Logging backends. -->
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
@@ -52,6 +58,25 @@
             <artifactId>janusgraph-inmemory</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <!-- needed by Cassandra ccm -->
+        <dependency>
+            <groupId>com.datastax.oss</groupId>
+            <artifactId>java-driver-test-infra</artifactId>
+            <version>${cassandra-driver.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.datastax.oss</groupId>
+                    <artifactId>java-driver-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-exec</artifactId>
+            <version>1.3</version>
+        </dependency>
+
     </dependencies>
 
     <profiles>
@@ -80,7 +105,6 @@
                             <argument>-classpath</argument>
                             <classpath/>
                             <argument>org.janusgraph.BenchmarkRunner</argument>
-                            <argument>.*Benchmark</argument>
                         </arguments>
                     </configuration>
                     </execution>

janusgraph-benchmark/src/main/java/org/janusgraph/BenchmarkRunner.java

@@ -14,6 +14,7 @@
 
 package org.janusgraph;
 
+import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge;
 import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
 import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.results.RunResult;
@@ -34,22 +35,7 @@
 import java.util.Map;
 
 public class BenchmarkRunner {
-    public static void main(String[] args) throws RunnerException, IOException {
-        final ChainedOptionsBuilder builder = new OptionsBuilder()
-            .forks(1)
-            .measurementTime(TimeValue.seconds(5))
-            .shouldFailOnError(true)
-            .warmupIterations(2)
-            .warmupTime(TimeValue.seconds(1));
-        if (args.length > 0) {
-            for (String arg : args) {
-                builder.include(arg);
-            }
-        } else {
-            builder.include(".*Benchmark");
-        }
-        builder.exclude(".*StaticArrayEntryListBenchmark");
-        Collection<RunResult> results = new Runner(builder.build()).run();
+    private static void transformResults(Collection<RunResult> results, List<Map<String, Object>> outputs) {
         Map<String, Double> metrics = new HashMap<>();
         for (RunResult result : results) {
             Result primaryResult = result.getPrimaryResult();
@@ -62,17 +48,68 @@ public static void main(String[] args) throws RunnerException, IOException {
             // we sum up results with different params against same benchmark
             metrics.put(benchmark, metrics.getOrDefault(benchmark, 0d) + score);
         }
-        List<Map<String, Object>> outputs = new ArrayList<>();
         for (Map.Entry<String, Double> entry : metrics.entrySet()) {
             outputs.add(new HashMap() {{
                 put("name", entry.getKey());
                 put("value", entry.getValue());
                 put("unit", "ms/op");
             }});
         }
-        String workspace = System.getenv("${GITHUB_WORKSPACE}");
-        File file = new File(workspace, "benchmark.json");
-        System.out.println("writing to " + file.getAbsolutePath());
+    }
+
+    private static void runCqlBenchmarks(ChainedOptionsBuilder cqlBuilder, List<Map<String, Object>> outputs) throws RunnerException, InterruptedException, IOException {
+        CcmBridge.Builder builder =
+            CcmBridge.builder()
+                .withNodes(1);
+        CcmBridge bridge = builder.build();
+        Runtime.getRuntime().addShutdownHook(
+            new Thread(
+                () -> {
+                    bridge.stop();
+                    bridge.remove();
+                }
+            )
+        );
+        bridge.create();
+        bridge.start();
+        System.out.println("Cassandra version = " + bridge.getCassandraVersion());
+
+        cqlBuilder.include("CQL.*Benchmark");
+        transformResults(new Runner(cqlBuilder.build()).run(), outputs);
+    }
+
+    private static ChainedOptionsBuilder getJmhBuilder() {
+        return new OptionsBuilder()
+            .forks(1)
+            .measurementTime(TimeValue.seconds(5))
+            .shouldFailOnError(true)
+            .warmupIterations(2)
+            .warmupTime(TimeValue.seconds(1));
+    }
+
+    public static void main(String[] args) throws RunnerException, IOException, InterruptedException {
+        final boolean runSpecifiedTests = args.length > 0;
+        final List<Map<String, Object>> outputs = new ArrayList<>();
+
+        final ChainedOptionsBuilder builder = getJmhBuilder();
+        if (runSpecifiedTests) {
+            for (String arg : args) {
+                builder.include(arg);
+            }
+        } else {
+            builder.include(".*Benchmark");
+            builder.exclude("StaticArrayEntryListBenchmark");
+            builder.exclude("CQL.*Benchmark");
+        }
+        Collection<RunResult> results = new Runner(builder.build()).run();
+        transformResults(results, outputs);
+
+        // run benchmarks using Cassandra (if ccm is available)
+        if (!runSpecifiedTests) {
+            runCqlBenchmarks(getJmhBuilder(), outputs);
+        }
+
+        File file = new File("benchmark.json");
         ObjectMapper objectMapper = new ObjectMapper();
         String json = objectMapper.writeValueAsString(outputs);
         BufferedWriter writer = new BufferedWriter(new FileWriter(file));

janusgraph-benchmark/src/main/java/org/janusgraph/CQLMultiQueryBenchmark.java

@@ -0,0 +1,103 @@
+// Copyright 2022 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraphTransaction;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
+import org.janusgraph.diskstorage.configuration.WriteConfiguration;
+import org.janusgraph.diskstorage.cql.CQLConfigOptions;
+import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@Fork(1)
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+public class CQLMultiQueryBenchmark {
+    @Param({"100", "500"})
+    int fanoutFactor;
+
+    JanusGraph graph;
+
+    public WriteConfiguration getConfiguration() {
+        ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration();
+        config.set(GraphDatabaseConfiguration.STORAGE_BACKEND,"cql");
+        config.set(CQLConfigOptions.LOCAL_DATACENTER, "dc1");
+        config.set(GraphDatabaseConfiguration.USE_MULTIQUERY, true);
+        return config.getConfiguration();
+    }
+
+    @Setup
+    public void setUp() throws Exception {
+        graph = JanusGraphFactory.open(getConfiguration());
+
+        JanusGraphManagement mgmt = graph.openManagement();
+        PropertyKey name = mgmt.makePropertyKey("name").dataType(String.class).make();
+        mgmt.buildIndex("nameIndex", Vertex.class).addKey(name).buildCompositeIndex();
+        mgmt.commit();
+
+        Vertex v = graph.addVertex("name", "outer");
+        for (int i = 0; i < fanoutFactor; i++) {
+            Vertex otherV = graph.addVertex("name", "middle");
+            v.addEdge("connects", otherV);
+            for (int j = 0; j < fanoutFactor; j++) {
+                Vertex innerV = graph.addVertex("name", "inner");
+                otherV.addEdge("connects", innerV);
+            }
+            graph.tx().commit();
+        }
+    }
+
+    @TearDown
+    public void tearDown() throws BackendException {
+        JanusGraphFactory.drop(graph);
+    }
+
+    @Benchmark
+    public List<Object> getNeighborNames() {
+        JanusGraphTransaction tx = graph.buildTransaction()
+            .start();
+        List<Object> names = tx.traversal().V().has("name", "outer").out().out().values("name").toList();
+        tx.rollback();
+        return names;
+    }
+
+    @Benchmark
+    public List<Object> getNames() {
+        JanusGraphTransaction tx = graph.buildTransaction()
+            .start();
+        List<Object> names = tx.traversal().V().has("name", "inner").values("name").toList();
+        tx.rollback();
+        return names;
+    }
+}