Upload sccache stats into benchmark database with build step time (pytorch#140839)

Guinea pig benchmark database
Pull Request resolved: pytorch#140839
Approved by: https://github.com/huydhn

Co-authored-by: Huy Do <[email protected]>

Author: 2 people

.github/actions/upload-sccache-stats/action.yml

@@ -0,0 +1,39 @@
+# Upload sccache stats to artifacts, and also as benchmark data when on an aws
+# linux or windows machine.  Does not currently handle mac builds
+name: Upload sccache stats
+
+description: Upload sccache stats to artifacts
+
+inputs:
+  github-token:
+    description: GITHUB_TOKEN
+    required: true
+  build-time:
+    description: Build time in seconds
+
+runs:
+  using: composite
+  steps:
+    - name: Upload sccache to s3
+      uses: seemethere/upload-artifact-s3@v5
+      with:
+        s3-prefix: |
+          ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact
+        retention-days: 14
+        if-no-files-found: warn
+        path: sccache-stats-*.json
+
+    - name: Format sccache stats
+      shell: bash
+      run: |
+        python3 -m tools.stats.sccache_stats_to_benchmark_format
+      env:
+        BUILD_TIME: ${{ inputs.build-time }}
+
+    - name: Upload sccache stats as benchmark
+      uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
+      with:
+        benchmark-results-dir: test/test-reports
+        dry-run: false
+        schema-version: v3
+        github-token: ${{ inputs.github-token }}

.github/workflows/_linux-build.yml

@@ -212,6 +212,7 @@ jobs:
           SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
           USE_SPLIT_BUILD: ${{ inputs.use_split_build }}
         run: |
+          START_TIME=$(date +%s)
           if [[ ${BUILD_ENVIRONMENT} == *"s390x"* ]]; then
             JENKINS_USER=
             USED_IMAGE="${DOCKER_IMAGE_S390X}"
@@ -256,6 +257,9 @@ jobs:
           )
           docker exec -t "${container_name}" sh -c '.ci/pytorch/build.sh'
 
+          END_TIME=$(date +%s)
+          echo "build_time=$((END_TIME - START_TIME))" >> "$GITHUB_OUTPUT"
+
       - name: Archive artifacts into zip
         if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped'
         run: |
@@ -301,14 +305,10 @@ jobs:
 
       - name: Upload sccache stats
         if: steps.build.outcome != 'skipped' && inputs.build-environment != 'linux-s390x-binary-manywheel'
-        uses: seemethere/upload-artifact-s3@v5
+        uses: ./.github/actions/upload-sccache-stats
         with:
-          s3-prefix: |
-            ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact
-          retention-days: 365
-          if-no-files-found: warn
-          path: sccache-stats-*.json
-          s3-bucket: ${{ inputs.s3-bucket }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          build-time: ${{ steps.build.outputs.build_time }}
 
       - name: Teardown Linux
         uses: pytorch/test-infra/.github/actions/teardown-linux@main

.github/workflows/_win-build.yml

@@ -182,13 +182,9 @@ jobs:
 
       - name: Upload sccache stats
         if: steps.build.outcome != 'skipped'
-        uses: seemethere/upload-artifact-s3@v5
+        uses: ./.github/actions/upload-sccache-stats
         with:
-          s3-prefix: |
-            ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact
-          retention-days: 14
-          if-no-files-found: warn
-          path: sccache-stats-*.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Teardown Windows
         uses: ./.github/actions/teardown-win

tools/stats/sccache_stats_to_benchmark_format.py

@@ -0,0 +1,63 @@
+import glob
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+def flatten_data(d: Dict[str, Any]) -> Dict[str, Any]:
+    # Flatten the sccache stats data from a possibly nested dictionary to a flat
+    # dictionary.  For example, the input:
+    # {
+    #     "cache": {
+    #         "hit": 1,
+    #         "miss": 2,
+    #     },
+    # }
+    # will be transformed to:
+    # {
+    #     "cache_hit": 1,
+    #     "cache_miss": 2,
+    # }
+    flat_data = {}
+    for key, value in d.items():
+        if isinstance(value, dict):
+            for k, v in flatten_data(value).items():
+                flat_data[f"{key}_{k}"] = v
+        else:
+            flat_data[key] = value
+    return flat_data
+
+
+def main() -> None:
+    records = []
+    for file in glob.glob(str(REPO_ROOT / "sccache-stats-*.json")):
+        with open(file) as f:
+            data = json.load(f)
+
+            # I don't know what sccache info will be most useful yet, and the
+            # sccache json has a decent number of keys, so just flatten the data
+            # and store all of it
+            records.append(
+                {
+                    "benchmark": {
+                        "name": "sccache_stats",
+                    },
+                    "metric": {
+                        "name": "sccache_stats",
+                        "benchmark_values": [int(os.environ.get("BUILD_TIME") or 0)],
+                        "extra_info": flatten_data(data),
+                    },
+                }
+            )
+    output_file = REPO_ROOT / "test" / "test-reports" / "sccache-stats.json"
+    os.makedirs(output_file.parent, exist_ok=True)
+    with open(output_file, "w") as f:
+        json.dump(records, f)
+
+
+if __name__ == "__main__":
+    main()