Run mongodb-benchmarking

.github/workflows/dance.yml

@@ -52,6 +52,8 @@ jobs:
           - { config: mongo-tools }
           - { config: mongo-core-test, verbose: true } # verbose to view test output on CI
 
+          - { config: mongobench-runall }
+
           - { config: ycsb-workloada }
           - { config: ycsb-workloada2 }
           - { config: ycsb-workloadb }

.gitignore

@@ -3,3 +3,6 @@
 /dumps/mongodump_tests/
 /vendor/
 cover.txt
+
+# mongodb-benchmarking
+projects/benchmark_results_*.csv

cmd/dance/main.go

@@ -40,6 +40,7 @@ import (
 	"github.com/FerretDB/dance/internal/runner"
 	"github.com/FerretDB/dance/internal/runner/command"
 	"github.com/FerretDB/dance/internal/runner/gotest"
+	"github.com/FerretDB/dance/internal/runner/mongobench"
 	"github.com/FerretDB/dance/internal/runner/ycsb"
 )
 
@@ -196,6 +197,8 @@ func main() {
 				runner, err = command.New(c.Params.(*config.RunnerParamsCommand), rl, cli.Verbose)
 			case config.RunnerTypeGoTest:
 				runner, err = gotest.New(c.Params.(*config.RunnerParamsGoTest), rl, cli.Verbose)
+			case config.RunnerTypeMongoBench:
+				runner, err = mongobench.New(c.Params.(*config.RunnerParamsMongoBench), rl)
 			case config.RunnerTypeYCSB:
 				runner, err = ycsb.New(c.Params.(*config.RunnerParamsYCSB), rl)
 			default:

internal/config/runner.go

@@ -24,6 +24,9 @@ const (
 	// RunnerTypeGoTest indicates a Go test runner.
 	RunnerTypeGoTest RunnerType = "gotest"
 
+	// RunnerTypeMongoBench indicates a MongoDB benchmark test runner.
+	RunnerTypeMongoBench RunnerType = "mongobench"
+
 	// RunnerTypeYCSB indicates a YCSB test runner.
 	RunnerTypeYCSB RunnerType = "ycsb"
 )
@@ -61,6 +64,15 @@ type RunnerParamsGoTest struct {
 // runnerParams implements [RunnerParams] interface.
 func (rp *RunnerParamsGoTest) runnerParams() {}
 
+// RunnerParamsMongoBench represents `mongobench` runner parameters.
+type RunnerParamsMongoBench struct {
+	Dir  string
+	Args []string
+}
+
+// runnerParams implements [RunnerParams] interface.
+func (rp *RunnerParamsMongoBench) runnerParams() {}
+
 // RunnerParamsYCSB represents `ycsb` runner parameters.
 type RunnerParamsYCSB struct {
 	Dir  string
@@ -74,5 +86,6 @@ func (rp *RunnerParamsYCSB) runnerParams() {}
 var (
 	_ RunnerParams = (*RunnerParamsCommand)(nil)
 	_ RunnerParams = (*RunnerParamsGoTest)(nil)
+	_ RunnerParams = (*RunnerParamsMongoBench)(nil)
 	_ RunnerParams = (*RunnerParamsYCSB)(nil)
 )

internal/configload/configload.go

@@ -174,6 +174,8 @@ func loadContent(content, db string) (*config.Config, error) {
 		p = &runnerParamsCommand{}
 	case config.RunnerTypeGoTest:
 		p = &runnerParamsGoTest{}
+	case config.RunnerTypeMongoBench:
+		p = &runnerParamsMongoBench{}
 	case config.RunnerTypeYCSB:
 		p = &runnerParamsYCSB{}
 	default:

internal/configload/runner.go

@@ -74,6 +74,20 @@ func (rp *runnerParamsGoTest) convert() (config.RunnerParams, error) {
 	}, nil
 }
 
+// runnerParamsMongoBench represents `mongobench` runner parameters in the project configuration YAML file.
+type runnerParamsMongoBench struct {
+	Dir  string   `yaml:"dir"`
+	Args []string `yaml:"args"`
+}
+
+// convert implements [runnerParams] interface.
+func (rp *runnerParamsMongoBench) convert() (config.RunnerParams, error) {
+	return &config.RunnerParamsMongoBench{
+		Dir:  rp.Dir,
+		Args: rp.Args,
+	}, nil
+}
+
 // runnerParamsYCSB represents `ycsb` runner parameters in the project configuration YAML file.
 type runnerParamsYCSB struct {
 	Dir  string   `yaml:"dir"`
@@ -92,5 +106,6 @@ func (rp *runnerParamsYCSB) convert() (config.RunnerParams, error) {
 var (
 	_ runnerParams = (*runnerParamsCommand)(nil)
 	_ runnerParams = (*runnerParamsGoTest)(nil)
+	_ runnerParams = (*runnerParamsMongoBench)(nil)
 	_ runnerParams = (*runnerParamsYCSB)(nil)
 )

internal/runner/mongobench/mongobench.go

@@ -0,0 +1,216 @@
+// Copyright 2021 FerretDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mongobench provides `mongobench` runner.
+package mongobench
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/FerretDB/dance/internal/config"
+	"github.com/FerretDB/dance/internal/runner"
+)
+
+// mongoBench represents `mongoBench` runner.
+type mongoBench struct {
+	p *config.RunnerParamsMongoBench
+	l *slog.Logger
+}
+
+// New creates a new runner with given parameters.
+func New(params *config.RunnerParamsMongoBench, l *slog.Logger) (runner.Runner, error) {
+	return &mongoBench{
+		p: params,
+		l: l,
+	}, nil
+}
+
+// parseFileNames parses the file names that store benchmark results.
+// Each operation is stored in different files such as `benchmark_results_insert.csv`,
+// `benchmark_results_update.csv`, `benchmark_results_delete.csv` and `benchmark_results_upsert.csv`.
+func parseFileNames(r io.Reader) ([]string, error) {
+	var fileNames []string
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if !strings.HasPrefix(line, "Benchmarking completed. Results saved to ") {
+			continue
+		}
+
+		// parse file name from the line such as `Benchmarking completed. Results saved to benchmark_results_delete.csv`
+		fileName := strings.TrimSpace(strings.TrimPrefix(line, "Benchmarking completed. Results saved to "))
+		fileNames = append(fileNames, fileName)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	if len(fileNames) == 0 {
+		return nil, errors.New("no benchmark result files found")
+	}
+
+	return fileNames, nil
+}
+
+// readResult reads the file and gets the last measurement and parses it.
+// The file contains measurements taken each second while the benchmark was running,
+// the last measurement is parsed and returned.
+func readResult(filePath string) (result map[string]float64, err error) {
+	var f *os.File
+
+	if f, err = os.Open(filePath); err != nil {
+		return
+	}
+
+	defer func() {
+		if e := f.Close(); e != nil && err == nil {
+			err = e
+		}
+	}()
+
+	// cannot use [csv.NewReader] because the file does not contain valid CSV,
+	// it contains 7 header fields while record lines contain 6 fields,
+	// so we parse it manually and assume the last field `mean_rate` is missing
+	s := bufio.NewScanner(f)
+
+	var lastLine string
+
+	for s.Scan() {
+		line := s.Text()
+
+		if strings.TrimSpace(line) != "" {
+			lastLine = line
+		}
+	}
+
+	if err = s.Err(); err != nil {
+		return
+	}
+
+	record := strings.Split(lastLine, ",")
+	if len(record) < 6 {
+		return nil, errors.New("insufficient fields")
+	}
+
+	var count, mean, m1Rate, m5Rate, m15Rate float64
+
+	if count, err = strconv.ParseFloat(record[1], 64); err != nil {
+		return
+	}
+
+	if mean, err = strconv.ParseFloat(record[2], 64); err != nil {
+		return
+	}
+
+	if m1Rate, err = strconv.ParseFloat(record[3], 64); err != nil {
+		return
+	}
+
+	if m5Rate, err = strconv.ParseFloat(record[4], 64); err != nil {
+		return
+	}
+
+	if m15Rate, err = strconv.ParseFloat(record[5], 64); err != nil {
+		return
+	}
+
+	result = map[string]float64{
+		"count":    count,
+		"mean":     mean,
+		"m1_rate":  m1Rate,
+		"m5_rate":  m5Rate,
+		"m15_rate": m15Rate,
+	}
+
+	return
+}
+
+// run runs given command in the given directory and returns parsed results.
+func run(ctx context.Context, args []string, dir string) (map[string]config.TestResult, error) {
+	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
+	cmd.Dir = dir
+	cmd.Stderr = os.Stderr
+
+	pipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	defer pipe.Close()
+
+	if err = cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	fileNames, err := parseFileNames(io.TeeReader(pipe, os.Stdout))
+	if err != nil {
+		_ = cmd.Process.Kill()
+		_ = cmd.Wait()
+
+		return nil, err
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return nil, err
+	}
+
+	res := make(map[string]config.TestResult)
+
+	for _, fileName := range fileNames {
+		var m map[string]float64
+
+		if m, err = readResult(filepath.Join("..", "projects", fileName)); err != nil {
+			return nil, err
+		}
+
+		op := strings.TrimSuffix(strings.TrimPrefix(fileName, "benchmark_results_"), ".csv")
+		res[op] = config.TestResult{
+			Status:       config.Pass,
+			Measurements: m,
+		}
+	}
+
+	return res, nil
+}
+
+// Run implements [runner.Runner] interface.
+func (m *mongoBench) Run(ctx context.Context) (map[string]config.TestResult, error) {
+	bin := filepath.Join("..", "bin", "mongodb-benchmarking")
+	if _, err := os.Stat(bin); err != nil {
+		return nil, err
+	}
+
+	bin, err := filepath.Abs(bin)
+	if err != nil {
+		return nil, err
+	}
+
+	args := append([]string{bin}, m.p.Args...)
+
+	m.l.InfoContext(ctx, "Run", slog.String("cmd", strings.Join(args, " ")))
+
+	return run(ctx, args, m.p.Dir)
+}