diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml index b93a11ecc2..6ab44296bf 100644 --- a/eng/ci/host.benchmarks.yml +++ b/eng/ci/host.benchmarks.yml @@ -12,6 +12,14 @@ schedules: - dev always: true +parameters: + +# Crank arguments customization. +- name: crankAdditionalArguments + displayName: Additional Crank arguments. For collecting perf trace, add --hostruntime.dotnetTrace true + type: string + default: '--variable duration=180 --variable warmup=60' + resources: repositories: - repository: 1es @@ -54,11 +62,10 @@ extends: - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self parameters: functionAppName: HelloHttpNet9NoProxy - # LINUX - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self parameters: - functionAppName: HelloHttpNet9 + functionAppName: HelloHttpNet9 os: Linux - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self parameters: @@ -74,10 +81,12 @@ extends: parameters: description: .NET9 Web Application functionAppName: HelloHttpNet9 # App with ASP.NET Integration + crankAdditionalArguments: ${{ parameters.crankAdditionalArguments }} - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: description: .NET9 Worker Application functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration + crankAdditionalArguments: ${{ parameters.crankAdditionalArguments }} # LINUX - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self @@ -85,8 +94,10 @@ extends: description: .NET9 Web Application functionAppName: HelloHttpNet9 os: Linux + crankAdditionalArguments: ${{ parameters.crankAdditionalArguments }} - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: description: .NET9 Worker Application functionAppName: HelloHttpNet9NoProxy os: Linux + crankAdditionalArguments: ${{ parameters.crankAdditionalArguments }} diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml index a25c19683a..c641df53e4 100644 --- a/eng/ci/templates/official/jobs/run-benchmarks.yml +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -9,6 +9,8 @@ parameters: values: - Windows - Linux +- name: crankAdditionalArguments + type: string jobs: - job: ${{ parameters.functionAppName }}_${{ parameters.os }} @@ -30,13 +32,15 @@ jobs: functionApp: ${{ parameters.functionAppName }} benchmarkArtifactName: benchmark_results_$(Agent.OS)_$(functionApp) functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) - benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json + traceDirectory: $(Build.ArtifactStagingDirectory)/out + benchmarkResultsJsonPath: $(traceDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json functionsWorkerRuntime: 'dotnet-isolated' configFilePath: "./eng/perf/http.benchmarks.yml" hostLocation: "./../../" baselineBenchmarkResultFilePath: '' baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult appAgentHostName: '' + crankExtraArguments: ${{ parameters.crankAdditionalArguments }} templateContext: inputs: @@ -45,12 +49,13 @@ jobs: pipeline: BaselineResult targetPath: $(baselineBenchmarkResultsDownloadDir) - outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputParentDirectory: $(traceDirectory) outputs: - - output: pipelineArtifact - displayName: Publish benchmark results - path: $(benchmarkResultsJsonPath) - artifact: $(benchmarkArtifactName) + - output: pipelineArtifact + displayName: Publish artifacts + path: $(traceDirectory) + artifact: AzFunc_$(Build.BuildNumber)_${{ parameters.os }}_${{ parameters.functionAppName }} + condition: always() steps: @@ -62,6 +67,10 @@ jobs: - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" displayName: Install Microsoft.Crank.Controller tool + - pwsh: | + New-Item -ItemType Directory -Path $(traceDirectory) + displayName: Create artifact directories + - ${{ if eq(parameters.os, 'Windows') }}: - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' displayName: Start crank-agent @@ -112,8 +121,9 @@ jobs: scriptLocation: inlineScript inlineScript: | $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)" + $crankArgs += " --hostruntime.options.traceOutput $(traceDirectory)/$(Build.BuildNumber)_$(functionApp)_$(Agent.OS)" $crankArgs += " --variable CrankAgentAppVm=$(appAgentHostName) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentHostName):5000" - $crankArgs += " $(AdditionalCrankArguments)" + $crankArgs += " $(crankExtraArguments)" $command = "crank $crankArgs" if ('$(storeBenchmarkResultsInDatabase)' -eq 'true') { $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`"" diff --git a/eng/ci/templates/official/jobs/run-coldstart.yml b/eng/ci/templates/official/jobs/run-coldstart.yml index caddd632c8..7344706676 100644 --- a/eng/ci/templates/official/jobs/run-coldstart.yml +++ b/eng/ci/templates/official/jobs/run-coldstart.yml @@ -95,7 +95,7 @@ jobs: displayName: Publish Trace artifacts path: $(traceDirectory) artifact: AzFunc_${{ parameters.os }}_${{ parameters.functionAppName }} - condition:: always() + condition: always() steps: