diff --git a/.github/workflows/dance.yml b/.github/workflows/dance.yml index 54a9313b6..07253a085 100644 --- a/.github/workflows/dance.yml +++ b/.github/workflows/dance.yml @@ -52,6 +52,8 @@ jobs: - { config: mongo-tools } - { config: mongo-core-test, verbose: true } # verbose to view test output on CI + - { config: mongobench-runall } + - { config: ycsb-workloada } - { config: ycsb-workloada2 } - { config: ycsb-workloadb } diff --git a/.gitignore b/.gitignore index 5e8b49efd..577181686 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ /dumps/mongodump_tests/ /vendor/ cover.txt + +# mongodb-benchmarking +projects/benchmark_results_*.csv diff --git a/cmd/dance/main.go b/cmd/dance/main.go index 8d72c53c6..acf98b379 100644 --- a/cmd/dance/main.go +++ b/cmd/dance/main.go @@ -40,6 +40,7 @@ import ( "github.com/FerretDB/dance/internal/runner" "github.com/FerretDB/dance/internal/runner/command" "github.com/FerretDB/dance/internal/runner/gotest" + "github.com/FerretDB/dance/internal/runner/mongobench" "github.com/FerretDB/dance/internal/runner/ycsb" ) @@ -196,6 +197,8 @@ func main() { runner, err = command.New(c.Params.(*config.RunnerParamsCommand), rl, cli.Verbose) case config.RunnerTypeGoTest: runner, err = gotest.New(c.Params.(*config.RunnerParamsGoTest), rl, cli.Verbose) + case config.RunnerTypeMongoBench: + runner, err = mongobench.New(c.Params.(*config.RunnerParamsMongoBench), rl) case config.RunnerTypeYCSB: runner, err = ycsb.New(c.Params.(*config.RunnerParamsYCSB), rl) default: diff --git a/internal/config/runner.go b/internal/config/runner.go index ce6b497d4..070518ca4 100644 --- a/internal/config/runner.go +++ b/internal/config/runner.go @@ -24,6 +24,9 @@ const ( // RunnerTypeGoTest indicates a Go test runner. RunnerTypeGoTest RunnerType = "gotest" + // RunnerTypeMongoBench indicates a MongoDB benchmark test runner. + RunnerTypeMongoBench RunnerType = "mongobench" + // RunnerTypeYCSB indicates a YCSB test runner. RunnerTypeYCSB RunnerType = "ycsb" ) @@ -61,6 +64,15 @@ type RunnerParamsGoTest struct { // runnerParams implements [RunnerParams] interface. func (rp *RunnerParamsGoTest) runnerParams() {} +// RunnerParamsMongoBench represents `mongobench` runner parameters. +type RunnerParamsMongoBench struct { + Dir string + Args []string +} + +// runnerParams implements [RunnerParams] interface. +func (rp *RunnerParamsMongoBench) runnerParams() {} + // RunnerParamsYCSB represents `ycsb` runner parameters. type RunnerParamsYCSB struct { Dir string @@ -74,5 +86,6 @@ func (rp *RunnerParamsYCSB) runnerParams() {} var ( _ RunnerParams = (*RunnerParamsCommand)(nil) _ RunnerParams = (*RunnerParamsGoTest)(nil) + _ RunnerParams = (*RunnerParamsMongoBench)(nil) _ RunnerParams = (*RunnerParamsYCSB)(nil) ) diff --git a/internal/configload/configload.go b/internal/configload/configload.go index e234d0238..c2d4f0cc6 100644 --- a/internal/configload/configload.go +++ b/internal/configload/configload.go @@ -174,6 +174,8 @@ func loadContent(content, db string) (*config.Config, error) { p = &runnerParamsCommand{} case config.RunnerTypeGoTest: p = &runnerParamsGoTest{} + case config.RunnerTypeMongoBench: + p = &runnerParamsMongoBench{} case config.RunnerTypeYCSB: p = &runnerParamsYCSB{} default: diff --git a/internal/configload/runner.go b/internal/configload/runner.go index 84f1753aa..65a3a8d8e 100644 --- a/internal/configload/runner.go +++ b/internal/configload/runner.go @@ -74,6 +74,20 @@ func (rp *runnerParamsGoTest) convert() (config.RunnerParams, error) { }, nil } +// runnerParamsMongoBench represents `mongobench` runner parameters in the project configuration YAML file. +type runnerParamsMongoBench struct { + Dir string `yaml:"dir"` + Args []string `yaml:"args"` +} + +// convert implements [runnerParams] interface. +func (rp *runnerParamsMongoBench) convert() (config.RunnerParams, error) { + return &config.RunnerParamsMongoBench{ + Dir: rp.Dir, + Args: rp.Args, + }, nil +} + // runnerParamsYCSB represents `ycsb` runner parameters in the project configuration YAML file. type runnerParamsYCSB struct { Dir string `yaml:"dir"` @@ -92,5 +106,6 @@ func (rp *runnerParamsYCSB) convert() (config.RunnerParams, error) { var ( _ runnerParams = (*runnerParamsCommand)(nil) _ runnerParams = (*runnerParamsGoTest)(nil) + _ runnerParams = (*runnerParamsMongoBench)(nil) _ runnerParams = (*runnerParamsYCSB)(nil) ) diff --git a/internal/runner/mongobench/mongobench.go b/internal/runner/mongobench/mongobench.go new file mode 100644 index 000000000..54d4de910 --- /dev/null +++ b/internal/runner/mongobench/mongobench.go @@ -0,0 +1,216 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mongobench provides `mongobench` runner. +package mongobench + +import ( + "bufio" + "context" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/FerretDB/dance/internal/config" + "github.com/FerretDB/dance/internal/runner" +) + +// mongoBench represents `mongoBench` runner. +type mongoBench struct { + p *config.RunnerParamsMongoBench + l *slog.Logger +} + +// New creates a new runner with given parameters. +func New(params *config.RunnerParamsMongoBench, l *slog.Logger) (runner.Runner, error) { + return &mongoBench{ + p: params, + l: l, + }, nil +} + +// parseFileNames parses the file names that store benchmark results. +// Each operation is stored in different files such as `benchmark_results_insert.csv`, +// `benchmark_results_update.csv`, `benchmark_results_delete.csv` and `benchmark_results_upsert.csv`. +func parseFileNames(r io.Reader) ([]string, error) { + var fileNames []string + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if !strings.HasPrefix(line, "Benchmarking completed. Results saved to ") { + continue + } + + // parse file name from the line such as `Benchmarking completed. Results saved to benchmark_results_delete.csv` + fileName := strings.TrimSpace(strings.TrimPrefix(line, "Benchmarking completed. Results saved to ")) + fileNames = append(fileNames, fileName) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + if len(fileNames) == 0 { + return nil, errors.New("no benchmark result files found") + } + + return fileNames, nil +} + +// readResult reads the file and gets the last measurement and parses it. +// The file contains measurements taken each second while the benchmark was running, +// the last measurement is parsed and returned. +func readResult(filePath string) (result map[string]float64, err error) { + var f *os.File + + if f, err = os.Open(filePath); err != nil { + return + } + + defer func() { + if e := f.Close(); e != nil && err == nil { + err = e + } + }() + + // cannot use [csv.NewReader] because the file does not contain valid CSV, + // it contains 7 header fields while record lines contain 6 fields, + // so we parse it manually and assume the last field `mean_rate` is missing + s := bufio.NewScanner(f) + + var lastLine string + + for s.Scan() { + line := s.Text() + + if strings.TrimSpace(line) != "" { + lastLine = line + } + } + + if err = s.Err(); err != nil { + return + } + + record := strings.Split(lastLine, ",") + if len(record) < 6 { + return nil, errors.New("insufficient fields") + } + + var count, mean, m1Rate, m5Rate, m15Rate float64 + + if count, err = strconv.ParseFloat(record[1], 64); err != nil { + return + } + + if mean, err = strconv.ParseFloat(record[2], 64); err != nil { + return + } + + if m1Rate, err = strconv.ParseFloat(record[3], 64); err != nil { + return + } + + if m5Rate, err = strconv.ParseFloat(record[4], 64); err != nil { + return + } + + if m15Rate, err = strconv.ParseFloat(record[5], 64); err != nil { + return + } + + result = map[string]float64{ + "count": count, + "mean": mean, + "m1_rate": m1Rate, + "m5_rate": m5Rate, + "m15_rate": m15Rate, + } + + return +} + +// run runs given command in the given directory and returns parsed results. +func run(ctx context.Context, args []string, dir string) (map[string]config.TestResult, error) { + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + + pipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + defer pipe.Close() + + if err = cmd.Start(); err != nil { + return nil, err + } + + fileNames, err := parseFileNames(io.TeeReader(pipe, os.Stdout)) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + + return nil, err + } + + if err = cmd.Wait(); err != nil { + return nil, err + } + + res := make(map[string]config.TestResult) + + for _, fileName := range fileNames { + var m map[string]float64 + + if m, err = readResult(filepath.Join("..", "projects", fileName)); err != nil { + return nil, err + } + + op := strings.TrimSuffix(strings.TrimPrefix(fileName, "benchmark_results_"), ".csv") + res[op] = config.TestResult{ + Status: config.Pass, + Measurements: m, + } + } + + return res, nil +} + +// Run implements [runner.Runner] interface. +func (m *mongoBench) Run(ctx context.Context) (map[string]config.TestResult, error) { + bin := filepath.Join("..", "bin", "mongodb-benchmarking") + if _, err := os.Stat(bin); err != nil { + return nil, err + } + + bin, err := filepath.Abs(bin) + if err != nil { + return nil, err + } + + args := append([]string{bin}, m.p.Args...) + + m.l.InfoContext(ctx, "Run", slog.String("cmd", strings.Join(args, " "))) + + return run(ctx, args, m.p.Dir) +} diff --git a/internal/runner/mongobench/mongobench_test.go b/internal/runner/mongobench/mongobench_test.go new file mode 100644 index 000000000..ad81cd367 --- /dev/null +++ b/internal/runner/mongobench/mongobench_test.go @@ -0,0 +1,91 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongobench + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseFileNames(t *testing.T) { + t.Parallel() + + //nolint:lll // verbatim output + output := strings.NewReader(` + +2025/05/26 15:28:13 Collection dropped. Starting new rate test... +2025/05/26 15:28:14 Timestamp: 1748240894, Document Count: 20138, Mean Rate: 20134.59 docs/sec, m1_rate: 0.00, m5_rate: 0.00, m15_rate: 0.00 +Benchmarking completed. Results saved to benchmark_results_insert.csv +2025/05/26 15:28:15 Starting update test... +2025/05/26 15:28:16 Timestamp: 1748240896, Document Count: 18717, Mean Rate: 18716.72 docs/sec, m1_rate: 0.00, m5_rate: 0.00, m15_rate: 0.00 +Benchmarking completed. Results saved to benchmark_results_update.csv +2025/05/26 15:28:16 Starting delete test... +2025/05/26 15:28:17 Timestamp: 1748240897, Document Count: 20690, Mean Rate: 20689.05 docs/sec, m1_rate: 0.00, m5_rate: 0.00, m15_rate: 0.00 +Benchmarking completed. Results saved to benchmark_results_delete.csv +2025/05/26 15:28:18 Collection dropped. Starting new rate test... +2025/05/26 15:28:19 Timestamp: 1748240899, Document Count: 13524, Mean Rate: 13522.22 docs/sec, m1_rate: 756.80, m5_rate: 756.80, m15_rate: 756.80 +2025/05/26 15:28:20 Timestamp: 1748240900, Document Count: 21505, Mean Rate: 10748.08 docs/sec, m1_rate: 756.80, m5_rate: 756.80, m15_rate: 756.80 +2025/05/26 15:28:21 Timestamp: 1748240901, Document Count: 27081, Mean Rate: 9027.36 docs/sec, m1_rate: 756.80, m5_rate: 756.80, m15_rate: 756.80 +Benchmarking completed. Results saved to benchmark_results_upsert.csv +`) + + actual, err := parseFileNames(output) + require.NoError(t, err) + + expected := []string{ + "benchmark_results_insert.csv", + "benchmark_results_update.csv", + "benchmark_results_delete.csv", + "benchmark_results_upsert.csv", + } + + assert.Equal(t, expected, actual) +} + +func TestReadResult(t *testing.T) { + t.Parallel() + + results := `t,count,mean,m1_rate,m5_rate,m15_rate,mean_rate +1748240899,13524,13522.216068,756.800000,756.800000,756.800000 +1748240900,21505,10748.078321,756.800000,756.800000,756.800000 +1748240901,27081,9027.363746,756.800000,756.800000,756.800000 +1748240902,30000,8288.694528,756.800000,756.800000,756.800000 +` + f, err := os.CreateTemp(t.TempDir(), "benchmark_results_delete") + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, f.Close()) + }) + + _, err = f.Write([]byte(results)) + require.NoError(t, err) + + actual, err := readResult(f.Name()) + require.NoError(t, err) + + expected := map[string]float64{ + "count": 30000, + "mean": 8288.694528, + "m1_rate": 756.800000, + "m5_rate": 756.800000, + "m15_rate": 756.800000, + } + assert.Equal(t, expected, actual) +} diff --git a/projects/mongobench-runall.yml b/projects/mongobench-runall.yml new file mode 100644 index 000000000..d25b59f46 --- /dev/null +++ b/projects/mongobench-runall.yml @@ -0,0 +1,26 @@ +--- +# Run all tests +runner: mongobench +params: + dir: . + args: + - -threads + - 10 + - -docs + - 100000 + - -uri + - {{.MONGODB_URI}} + - --runAll + +results: + mongodb: + stats: + pass: 4 + + ferretdb2: + stats: + pass: 4 + + ferretdb2-dev: + stats: + pass: 4