Merge branch 'master' into download-group-show-button

Author: flelayo

.circleci/config.yml

@@ -283,7 +283,7 @@ jobs:
             name: Run end-2-end tests with studies in local database
             environment:
               CBIOPORTAL_URL: http://localhost:8080
-              JUNIT_REPORT_PATH: ./local/junit
+              JUNIT_REPORT_PATH: ./local/junit/
               SCREENSHOT_DIRECTORY: ./local/screenshots
               PORTAL_SOURCE_DIR: /tmp/repos/cbioportal-frontend
               TEST_HOME: /tmp/repos/cbioportal-frontend/end-to-end-test/local
@@ -297,7 +297,8 @@ jobs:
             environment:
               TEST_HOME: /tmp/repos/cbioportal-frontend/end-to-end-test/local
             command: |
-              for f in $TEST_HOME/screenshots/reference/*.png; do
+              cd cbioportal-frontend
+              for f in end-to-end-test/local/screenshots/reference/*.png; do
                 git ls-files --error-unmatch $f > /dev/null 2> /dev/null ||
                 (echo -e "\033[0;31m $f not tracked \033[0m" && touch screenshots_not_tracked);
               done;
@@ -321,8 +322,8 @@ jobs:
             path: /tmp/repos/cbioportal-frontend/end-to-end-test/shared/imageCompare.html
             destination: /imageCompare.html
         - store_artifacts:
-            path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/customReport.json
-            destination: /customReport.json
+            path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/completeResults.json
+            destination: /completeResults.json
         - store_artifacts:
             path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/errors/
             destination: /errors
@@ -466,7 +467,6 @@ jobs:
         SUCCESS_COLOR: 'brightgreen'
         FAILURE_MESSAGE: 'Failing'
         FAILURE_COLOR: 'FF0A0A'
-        SEVERITY_THRESHOLD: 'CRITICAL'
         TEST_REPO_URL: '[email protected]:cBioPortal/cbioportal-test.git'
       steps:
         - attach_workspace:
@@ -487,7 +487,7 @@ jobs:
         - run:
             name: Count vulnerabilities and update status badge
             command: |
-              COUNT=$(jq '[.[] | select(.severity == "$SEVERITY_THRESHOLD")] | length' master_report.sbom)
+              COUNT=$(jq '[.[] | select(.severity == "CRITICAL" or .severity == "HIGH")] | length' master_report.sbom)
               cd cbioportal-test
               if [ $COUNT -eq 0 ]; then
                 jq --arg msg "$SUCCESS_MESSAGE" --arg col "$SUCCESS_COLOR" '.message = $msg | .color = $col' security-status.json > temp.json

.githooks/spotless_pre_commit_hook.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+HOOK_FILE=".git/hooks/pre-commit"
+
+# Ensure hooks directory exists
+mkdir -p .git/hooks
+
+# If the pre-commit hook doesn't exist, create it with a shebang line
+if [[ ! -f "$HOOK_FILE" ]]; then
+    echo "#!/bin/bash" > "$HOOK_FILE"
+    echo "set -e" >> "$HOOK_FILE"
+fi
+
+# Check if Spotless is already in the pre-commit hook, and add it if missing
+if ! grep -qxF "mvn spotless:apply" "$HOOK_FILE"; then
+    echo "echo 'Running Spotless on staged files...'" >> "$HOOK_FILE"
+    echo "mvn spotless:apply" >> "$HOOK_FILE"
+fi
+
+# Check if `git add -u` is already in the pre-commit hook, and add it if missing
+if ! grep -qxF "git add -u" "$HOOK_FILE"; then
+    echo "echo 'Adding formatted files back to commit...'" >> "$HOOK_FILE"
+    echo "git add -u" >> "$HOOK_FILE"
+fi
+
+# Make sure the pre-commit hook is executable
+chmod +x "$HOOK_FILE"
+
+echo "✅ Spotless pre-commit hook installed successfully!"

.github/workflows/core-test.yml

@@ -1,5 +1,19 @@
 name: Core tests
-on: [push, pull_request]
+
+on:
+  push:
+  pull_request_target:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch or PR to test (default: main)'
+        required: false
+        default: 'master'
+
+permissions:
+      id-token: write   # This is required for requesting the JWT
+      contents: read    # This is required for actions/checkout
+
 jobs:
   build:
     name: Setup and Test
@@ -12,6 +26,9 @@ jobs:
         ports:
           - 3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10
+    env:
+      TEST_DB_CLICKHOUSE_PASSWORD: ${{ secrets.TEST_DB_CLICKHOUSE_PASSWORD }}
+      TEST_DB_MYSQL_PASSWORD: ${{ secrets.TEST_DB_MYSQL_PASSWORD }}
     steps:
       - name: 'Checkout cbioportal repo'
         uses: actions/checkout@v2
@@ -64,4 +81,8 @@ jobs:
       - name: 'Run integration tests separately'
         working-directory: ./cbioportal
         run: |
-          mvn -U -Ppublic integration-test -Ddb.test.username=cbio_user -Ddb.test.password=somepassword
+          mvn verify -Pintegration-test
+      - name: 'Run e2e tests separately'
+        working-directory: ./cbioportal
+        run: |
+          mvn verify -Pe2e-test

.github/workflows/dockerimage.yml

@@ -3,12 +3,6 @@ on:
   push:
     branches:
       - master
-      - release-*
-      - rc
-      - rfc*
-      - demo-*
-      - fusion-sv-migration
-      - redis-branch-up-to-date
     tags: '*'
 
 jobs:
@@ -128,3 +122,81 @@ jobs:
             image: ${{ steps.meta.outputs.tags }}
             dependency-snapshot: 'true'
             github-token: '${{ secrets.GITHUB_TOKEN }}'
+
+  update_master_k8s_deployment:
+    needs: build_and_publish_web
+    if: github.repository == 'cBioPortal/cbioportal' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    env:
+      DEPLOYMENT_FILE_PATH: "argocd/aws/203403084713/clusters/cbioportal-prod/apps/cbioportal"
+      DEPLOYMENT_FILE_NAMES: "cbioportal_backend_master_blue.yaml cbioportal_backend_master_green.yaml"
+      K8S_REPO: 'knowledgesystems/knowledgesystems-k8s-deployment'
+      ARGOCD_CLI_VER: '2.13.3'
+      ARGOCD_SERVER: 'argocd.cbioportal.org'
+      ARGOCD_APP: 'cbioportal'
+      ARGOCD_RESOURCE_OPTIONS: '--resource apps:Deployment:default/cbioportal-backend-master-blue --resource apps:Deployment:default/cbioportal-backend-master-green'
+    steps:
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            cbioportal/cbioportal
+          # Do not generate latest tag
+          flavor: |
+            latest=false
+            suffix=-web-shenandoah
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+
+      - name: Extract Image Tag
+        run: |
+          DOCKER_TAG=${{ steps.meta.outputs.tags }}
+          docker pull $DOCKER_TAG
+          DOCKER_SHA=$(docker inspect --format="{{index .RepoDigests 0}}" $DOCKER_TAG)
+          echo "DOCKER_SHA=$DOCKER_SHA" >> $GITHUB_ENV
+
+      - name: Checkout K8s Repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.K8S_REPO }}
+          token: ${{ secrets.K8S_REPO_TOKEN }}
+
+      - name: Update Master Deployment File
+        run: |
+          for FILE in $DEPLOYMENT_FILE_NAMES; do
+            sed -i "s|image:.*|image: $DOCKER_SHA|" $DEPLOYMENT_FILE_PATH/$FILE
+          done
+
+      - name: Add and commit changes to deployment file
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "cBioPortal Backend Github Action"
+          if ! git diff --quiet; then
+            git add -A
+            git commit -m "Update master deployment"
+          else
+            echo "No changes to commit."
+          fi
+
+      - name: Push changes to K8s repo
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: '${{ secrets.K8S_REPO_TOKEN }}'
+          repository: ${{ env.K8S_REPO }}
+          branch: master
+
+      - name: Setup ArgoCD
+        uses: clowdhaus/argo-cd-action/@main
+        with:
+          version: ${{ env.ARGOCD_CLI_VER }}
+          command: login ${{ env.ARGOCD_SERVER }}
+          options: --username ${{ secrets.ARGOCD_2034_USERNAME }} --password ${{ secrets.ARGOCD_2034_PASSWORD }}
+
+      - name: Sync master deployment
+        uses: clowdhaus/argo-cd-action/@main
+        with:
+          version: ${{ env.ARGOCD_CLI_VER }}
+          command: app sync ${{ env.ARGOCD_APP }}
+          options: ${{ env.ARGOCD_RESOURCE_OPTIONS }}

.github/workflows/integration-test.yml

@@ -1,5 +1,5 @@
 name: Integration tests
-on: [push, pull_request]
+on: [push, pull_request_target]
 jobs:
   build:
     name: Integration tests
@@ -45,10 +45,10 @@ jobs:
           cat $PORTAL_SOURCE_DIR/src/main/resources/application.properties | \
               sed 's|spring.datasource.url=.*|spring.datasource.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
               sed 's|spring.datasource.username=.*|spring.datasource.username=cbio_user|' | \
-              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' \
+              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' | \
+              sed 's|session.service.url=.*|session.service.url=http://cbioportal-session:5001/api/sessions/my_portal/|' | \
+              sed 's|feature.study.export=.*|feature.study.export=true|' \
               > application.properties
-      - name: 'Copy cgds.sql file into Docker Compose'
-        run: cp ./cbioportal/src/main/resources/db-scripts/cgds.sql ./cbioportal-docker-compose/data/.
       - name: 'Dump Properties'
         working-directory: ./cbioportal-docker-compose
         run: cat config/application.properties
@@ -73,6 +73,11 @@ jobs:
         working-directory: ./cbioportal-docker-compose
         run: |
           $PORTAL_SOURCE_DIR/test/integration/test_load_study.sh
+      - name: 'TEST - Import and Export of study_es_0_import_export'
+        if: steps.startup.conclusion == 'success'
+        working-directory: ./cbioportal-docker-compose
+        run: |
+          $PORTAL_SOURCE_DIR/test/integration/test_import_export.sh
       - name: 'TEST - Add OncoKB annotations to study'
         if: steps.startup.conclusion == 'success'
         working-directory: ./cbioportal-docker-compose

.github/workflows/label-check.yml

@@ -32,20 +32,27 @@ jobs:
         id: check_labels
         run: |
           PR_NUMBER=$(jq -r ".number" $GITHUB_EVENT_PATH)
-          PR_LABELS=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | \
+          PR_LABELS_STRING=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | \
                        jq -r '.labels[].name')
+          mapfile -t PR_LABELS <<< "$PR_LABELS_STRING"
           mapfile -t AVAILABLE_LABELS < labels.txt
-          for LABEL in ${PR_LABELS[@]}; do
-            if [[ "$LABEL" == "skip-changelog" ]]; then
+          pr_label_index=0
+          while [ $pr_label_index -lt ${#PR_LABELS[@]} ] ; do
+            pr_label="${PR_LABELS[$pr_label_index]}"
+            if [[ "$pr_label" == "skip-changelog" ]] ; then
               echo "PR contains a valid label: skip-changelog"
               exit 0  # Valid label found, exit successfully
             fi
-            for AVAILABLE_LABEL in "${AVAILABLE_LABELS[@]}"; do
-              if [[ "$AVAILABLE_LABEL" == "$LABEL" ]]; then
-                echo "PR contains a valid label: $LABEL"
+            available_label_index=0
+            while [ $available_label_index -lt ${#AVAILABLE_LABELS[@]} ] ; do
+              available_label="${AVAILABLE_LABELS[$available_label_index]}"
+              if [[ "$available_label" == "$pr_label" ]] ; then
+                echo "PR contains a valid label: $pr_label"
                 exit 0  # Valid label found, exit successfully
               fi
+              available_label_index=$(($available_label_index+1))
             done
+            pr_label_index=$(($pr_label_index+1))
           done
           echo "No valid label found on PR."
           echo "Available label options from release-drafter.yml:"

.github/workflows/markdown-links-check.yml

@@ -0,0 +1,15 @@
+name: Markdown links check
+on: [pull_request]
+jobs:
+  build:
+    name: Markdown links check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        # Docs here https://github.com/tcort/github-action-markdown-link-check
+      - uses: tcort/github-action-markdown-link-check@v1
+        continue-on-error: true
+        with:
+          use-quiet-mode: 'yes'
+          use-verbose-mode: 'no'
+          config-file: markdown_links_check_config.json

.github/workflows/maven-package-build.yml

@@ -73,3 +73,34 @@ jobs:
           MAVEN_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
           MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_CENTRAL_GPG_PASSPHRASE }}
           ref: ${{ inputs.source_ref }}
+
+      - name: Check publish status
+        env:
+          ARTIFACT_ID: "cbioportal"
+          GROUP_ID_PATH: "io/github/cbioportal"
+          MAX_RETRIES: 30
+          INTERVAL_SECONDS: 60
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          VERSION=${{ inputs.source_ref }}
+          elif [ "${{ github.event_name }}" == "release" ]; then
+          VERSION=${{ github.event.release.tag_name }}
+          else
+          echo "Action triggered through a invalid event!";
+          exit 1
+          fi
+
+          URL="https://repo1.maven.org/maven2/${GROUP_ID_PATH}/${ARTIFACT_ID}/${VERSION}"
+
+          for ((i=1; i<=$MAX_RETRIES; i++)); do
+          if curl --silent --head --fail "$URL" > /dev/null; then
+          echo "Package successfully published!"
+          exit 0
+          else
+          echo "Package not yet published! Retrying check in $INTERVAL_SECONDS seconds..."
+          sleep "$INTERVAL_SECONDS"
+          fi
+          done
+
+          echo "Successful publish status NOT FOUND after $MAX_RETRIES attempts."
+          exit 1

.github/workflows/retype-action.yml

@@ -19,7 +19,7 @@ jobs:
         with:
           dotnet-version: 7.0.x
 
-      - uses: retypeapp/action-build@latest
+      - uses: retypeapp/action-build@ddc4449f15ae65b75e13198b88200bfd64aef168 # before v3.10.0, which breaks the news on homepage
 
       - uses: retypeapp/action-github-pages@latest
         with: