Bounty: Speedup CI setup to <20s using native install

.gitattributes

@@ -1,5 +1,4 @@
 * text=auto
-
 # to move existing files into LFS:
 # git add --renormalize .
 *.onnx filter=lfs diff=lfs merge=lfs -text
@@ -8,7 +7,6 @@
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
-
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
 system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
@@ -19,3 +17,8 @@ third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
 third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
 third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
 third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*.hevc filter=lfs diff=lfs merge=lfs -text
+*.qcom2 filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text

.github/workflows/auto_pr_review.yaml

@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: read
       pull-requests: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/badges.yaml

@@ -12,7 +12,7 @@ env:
 jobs:
   badges:
     name: create badges
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     permissions:
       contents: write

.github/workflows/ci_weekly_report.yaml

@@ -16,7 +16,7 @@ env:
 jobs:
   setup:
     if: github.repository == 'commaai/openpilot'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
     steps:
@@ -37,7 +37,7 @@ jobs:
 
   report:
     needs: [ci_matrix_run]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: always()
     steps:
       - name: Get job results

.github/workflows/docs.yaml

@@ -18,7 +18,7 @@ concurrency:
 jobs:
   docs:
     name: build docs
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-20.04
     steps:
     - uses: commaai/timeout@v1
 

.github/workflows/jenkins-pr-trigger.yaml

@@ -7,7 +7,7 @@ on:
 jobs:
   # TODO: gc old branches in a separate job in this workflow
   scan-comments:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: ${{ github.event.issue.pull_request }}
     steps:
     - name: Check for trigger phrase

.github/workflows/model_review.yaml

@@ -12,7 +12,7 @@ jobs:
     permissions:
       contents: read
       pull-requests: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout

.github/workflows/prebuilt.yaml

@@ -11,7 +11,7 @@ env:
 jobs:
   build_prebuilt:
     name: build prebuilt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     env:
       PUSH_IMAGE: true

.github/workflows/release.yaml

@@ -11,7 +11,7 @@ jobs:
       ImageOS: ubuntu24
     container:
       image: ghcr.io/commaai/openpilot-base:latest
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     permissions:
       checks: read

.github/workflows/repo-maintenance.yaml

@@ -12,7 +12,7 @@ env:
 
 jobs:
   update_translations:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     steps:
       - uses: actions/checkout@v4
@@ -34,7 +34,7 @@ jobs:
 
   package_updates:
     name: package_updates
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     container:
       image: ghcr.io/commaai/openpilot-base:latest
     if: github.repository == 'commaai/openpilot'

.github/workflows/selfdrive_tests.yaml

@@ -249,7 +249,7 @@ jobs:
 
   car_docs_diff:
     name: PR comments
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     #if: github.event_name == 'pull_request'
     if: false  # TODO: run this in opendbc?
     steps:

.github/workflows/setup/action.yaml

@@ -1,56 +1,67 @@
-name: 'openpilot env setup'
+name: 'openpilot env setup (native no-docker)'
 
 inputs:
   is_retried:
-    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    description: 'Mock param to maintain compatibility'
     required: false
     default: 'false'
 
 runs:
   using: "composite"
   steps:
-    # assert that this action is retried using the setup-with-retry
-    - shell: bash
-      if: ${{ inputs.is_retried == 'false' }}
-      run: |
-        echo "You should not run this action directly. Use setup-with-retry instead"
-        exit 1
-
-    - shell: bash
-      name: No retries!
+    - name: Show runner info
+      shell: bash
       run: |
-        if [ "${{ github.run_attempt }}" -gt 1 ]; then
-          echo -e "\033[0;31m##################################################"
-          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
-          echo -e "\033[0;31m##################################################\033[0m"
-          exit 1
-        fi
+        echo "Runner OS: $RUNNER_OS"
+        echo "Arch: $(uname -m)"
 
-    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
-    - shell: bash
-      run: git lfs pull
+    - name: Restore apt cache
+      uses: actions/cache@v3
+      with:
+        path: /var/cache/apt/archives
+        key: apt-cache-${{ runner.os }}
+        restore-keys: |
+          apt-cache-${{ runner.os }}
 
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
+    - name: Restore pip cache
+      uses: actions/cache@v3
       with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        path: ~/.cache/pip
+        key: pip-cache-${{ runner.os }}
         restore-keys: |
-          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
+          pip-cache-${{ runner.os }}
+
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        sudo apt update
+        sudo apt install -y \
+          git \
+          python3-pip \
+          python3-dev \
+          python3-venv \
+          build-essential \
+          libcap-dev \
+          libusb-1.0-0-dev \
+          libavcodec-dev \
+          libavformat-dev \
+          libswscale-dev \
+          libopencv-dev \
+          libeigen3-dev \
+          libprotobuf-dev \
+          protobuf-compiler \
+          libturbojpeg0-dev \
+          libcurl4-openssl-dev
+
+    - name: Set up Python environment
+      shell: bash
+      run: |
+        python3 -m venv venv
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+
+    - name: Install Python dependencies
       shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
       run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
-    # build our docker image
-    - shell: bash
-      run: eval ${{ env.BUILD }}
+        source venv/bin/activate
+        pip install -r requirements.txt

.github/workflows/stale.yaml

@@ -11,7 +11,7 @@ env:
 
 jobs:
   stale:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/stale@v9
         with:
@@ -32,7 +32,7 @@ jobs:
 
   # same as above, but give draft PRs more time
   stale_drafts:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/stale@v9
         with:

.github/workflows/ui_preview.yaml

@@ -22,7 +22,7 @@ jobs:
     #if: github.repository == 'commaai/openpilot'
     if: false  # FIXME: FrameReader is broken on CI runners
     name: preview
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 20
     permissions:
       contents: read

.lfsconfig

@@ -1,4 +1,2 @@
 [lfs]
-	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-	pushurl = ssh://[email protected]/commaai/openpilot-lfs.git
-	locksverify = false
+	url = https://github.com/nkhatpe/openpilot.git/info/lfs