openms-ci-full #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This does the whole cmake build of OpenMS on github's runners. It is intended to replace our current Jenkins based jobs. | |
| # For Nightly and Release branches it builds the packages and submits them to our archive. | |
| # NB: The actual upload is currently commented out so that this can get merged to develop. It will be re-added presently. | |
| name: openms-ci-full | |
| # Controls when the action will run. Triggers the workflow on push or pull request | |
| # events or manually via workflow_dispatch | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| package: | |
| type: boolean | |
| description: Build and upload packages/installer/documentation? | |
| default: false | |
| knime: | |
| type: boolean | |
| description: Build and upload KNIME plugins and update site? | |
| default: false | |
| do_release: | |
| type: boolean | |
| description: Make tags and create a github release? | |
| default: false | |
| mark_as_latest: | |
| type: boolean | |
| description: Update latest to point to this release | |
| default: false | |
| announce_release: | |
| type: boolean | |
| description: Update website and create announcement material for release | |
| default: false | |
| push: | |
| branches: | |
| - nightly | |
| - release/* | |
| pull_request: | |
| branches: | |
| - develop | |
| # A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
| jobs: | |
| build-and-test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: windows-2022 | |
| # cl.exe is currently the only supported. Needs a VS shell to be set up (especially if used without VS CMake Generator) | |
| # clang.exe is also installed but untested (might crash with our VS specific compiler definitions) | |
| # clang-cl.exe might be used instead. | |
| compiler: cl.exe | |
| # VS 2022: 14.30-14.39 (corresponds to MSVC v143) | |
| # VS 2019: 14.20-14.29 (corresponds to MSVC v142) | |
| compiler_ver: 14.36 # Specific toolset version for VS 2022 -- must (roughly) match Qt6 compiler (see below) | |
| - os: macos-13 | |
| # Since the appleclang version is dependent on the XCode versions that are installed | |
| # on the GH runners, use this as the choice for a specific version | |
| # TODO currently unused but XCode/appleclang is the default | |
| compiler: xcode | |
| # TODO implement support for other versions | |
| compiler_ver: 14.2 | |
| - os: macos-14 | |
| # Since the appleclang version is dependent on the XCode versions that are installed | |
| # on the GH runners, use this as the choice for a specific version | |
| # TODO currently unused but XCode/appleclang is the default | |
| compiler: xcode | |
| # TODO implement support for other versions | |
| compiler_ver: 15.0.1 | |
| # Make sure the compilers are available in that version on the GH runners | |
| # We are not installing anything. | |
| - os: ubuntu-22.04 | |
| compiler: g++ | |
| compiler_ver: 11 | |
| # Make sure the compilers are available in that version on the GH runners | |
| # We are not installing anything. | |
| #- os: ubuntu-22.04-arm | |
| # compiler: g++ | |
| # compiler_ver: 11 | |
| - os: ubuntu-22.04 | |
| compiler: clang++ | |
| compiler_ver: 15 | |
| outputs: | |
| version_number: ${{ steps.create_changelog.outputs.version_number }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| TERM: xterm-256color | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: OpenMS | |
| - id: extract_branch | |
| name: Extract branch/PR infos | |
| shell: bash | |
| run: echo "RUN_NAME=${{ github.event.pull_request && github.event.number || github.ref_name }}" >> $GITHUB_OUTPUT | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - id: set-vars | |
| name: Set extra variables | |
| run: | | |
| DO_PACKAGE=$( [[ ( "${{ inputs.package }}" || "${{ steps.extract_branch.outputs.RUN_NAME }}" == "nightly" || "${{ steps.extract_branch.outputs.RUN_NAME }}" =~ release/* ) && "${{ matrix.compiler }}" != "clang++" ]] && echo true || echo false) | |
| echo $DO_PACKAGE | |
| if [[ "${{ matrix.os }}" == ubuntu-* ]]; then | |
| echo "tp_folder=Linux" >> $GITHUB_OUTPUT | |
| echo "xvfb=xvfb-run -a" >> $GITHUB_OUTPUT | |
| echo "static_boost=OFF" >> $GITHUB_OUTPUT | |
| # always run doxygen on Ubuntu (because its fast), to detect Doxygen errors | |
| echo "enable_docs=ON" >> $GITHUB_OUTPUT | |
| if [ "$DO_PACKAGE" = true ]; then | |
| echo "pkg_type=deb" >> $GITHUB_OUTPUT | |
| else | |
| echo "pkg_type=none" >> $GITHUB_OUTPUT | |
| fi | |
| echo "cxx_compiler=${{ matrix.compiler }}-${{ matrix.compiler_ver }}" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ matrix.os }}" == windows-* ]]; then | |
| echo "tp_folder=Windows" >> $GITHUB_OUTPUT | |
| echo "contrib_os=windows" >> $GITHUB_OUTPUT | |
| echo "contrib_os_ver=" >> $GITHUB_OUTPUT | |
| echo "contrib_compiler=msvc-14.2/" >> $GITHUB_OUTPUT | |
| echo "xvfb=" >> $GITHUB_OUTPUT | |
| echo "static_boost=ON" >> $GITHUB_OUTPUT | |
| if [ "$DO_PACKAGE" = true ]; then | |
| echo "pkg_type=nsis" >> $GITHUB_OUTPUT | |
| echo "enable_docs=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "pkg_type=none" >> $GITHUB_OUTPUT | |
| echo "enable_docs=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| echo "cxx_compiler=${{ matrix.compiler }}" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ matrix.os }}" == macos-* ]]; then | |
| echo "tp_folder=MacOS" >> $GITHUB_OUTPUT | |
| echo "contrib_os=macOS" >> $GITHUB_OUTPUT | |
| echo "contrib_os_ver=10.15.5/" >> $GITHUB_OUTPUT | |
| echo "contrib_compiler=appleclang-11.0.0/" >> $GITHUB_OUTPUT | |
| echo "xvfb=" >> $GITHUB_OUTPUT | |
| echo "static_boost=OFF" >> $GITHUB_OUTPUT | |
| if [ "$DO_PACKAGE" = true ]; then | |
| echo "pkg_type=pkg" >> $GITHUB_OUTPUT | |
| echo "enable_docs=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "pkg_type=none" >> $GITHUB_OUTPUT | |
| echo "enable_docs=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ matrix.compiler }}" == "xcode" ]]; then | |
| sudo xcode-select -s '/Applications/Xcode_${{ matrix.compiler_ver }}.app/Contents/Developer' | |
| echo "cxx_compiler=clang++" >> $GITHUB_OUTPUT | |
| elif [[ "${{ matrix.compiler }}" == "clang++" ]]; then | |
| echo "cxx_compiler=$(brew --prefix llvm@${{ matrix.compiler_ver }})/bin/clang++" >> $GITHUB_OUTPUT | |
| else | |
| echo "Unsupported compiler" | |
| exit 1 | |
| fi | |
| fi | |
| shell: bash | |
| # Create the changelog for the release from our overall changelog | |
| - id: create_changelog | |
| if: matrix.compiler != 'clang++' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu') | |
| shell: bash | |
| name: Create changelog for release on Ubuntu + GCC | |
| run: | | |
| if [[ ${{ steps.extract_branch.outputs.RUN_NAME }} =~ ^release\/([0-9]+\.[0-9]+\.[0-9]+) ]]; then | |
| VERSION_NUMBER=${BASH_REMATCH[1]} | |
| echo "Version number: $VERSION_NUMBER" | |
| if [ "${VERSION_NUMBER##*.}" -eq "0" ] # Handle trailing 0 in major releases (eg 3.0.0) | |
| then | |
| VERSION_NUMBER=${VERSION_NUMBER%.*} | |
| fi | |
| echo "version_number=$VERSION_NUMBER" >> $GITHUB_OUTPUT | |
| grep -ne "----[[:space:]]*OpenMS" ${{ github.workspace }}/OpenMS/CHANGELOG > index_changelog.txt | |
| START=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | head -1) | |
| END=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | tail -1) | |
| echo "Extracting between lines:" | |
| echo $START | |
| echo $END | |
| awk "NR > $START && NR < $END" OpenMS/CHANGELOG > ${{ github.workspace }}/changelog.txt | |
| else | |
| echo "Branch name does not match the expected pattern." | |
| exit 1 | |
| fi | |
| - name: check input options | |
| if: github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: | | |
| # check if we want to be marked as latest, and if we actually should. | |
| if [[ 'true' == ${{ inputs.do_release }} ]]; then | |
| if [[ 'false' == ${{ inputs.knime }} || 'false' == ${{ inputs.package }} ]]; then | |
| echo "Can't do a release without building the OpenMS package and the KNIME package" | |
| exit 1 | |
| else | |
| if [[ 'true' == ${{ inputs.mark_as_latest }} && "$(git tag -l ${{ steps.create_changelog.outputs.version_number }})" ]]; then | |
| echo "Can't mark as latest, a tag with the same version already exists" | |
| exit 1 | |
| fi | |
| fi | |
| else | |
| if [[ 'true' == ${{ inputs.mark_as_latest }} || 'true' == ${{inputs.announce_release}} ]]; then | |
| echo "Can't mark as latest or announce release unless we do a release" | |
| exit 1 | |
| fi | |
| fi | |
| - name: Emulate a Visual Studio shell (Windows) | |
| if: startsWith(matrix.os, 'windows') | |
| uses: egor-tensin/vs-shell@v2 | |
| with: | |
| # NOTE: x64 is hardcoded. No support for 32bit | |
| arch: x64 | |
| - name: Setup conda environment for Parquet (Windows only) | |
| if: startsWith(matrix.os, 'windows') | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| auto-update-conda: true | |
| python-version: 3.9 | |
| activate-environment: openms-parquet | |
| use-mamba: true | |
| - name: Install Parquet dependencies (Windows only) | |
| if: startsWith(matrix.os, 'windows') | |
| shell: bash -el {0} | |
| run: | | |
| conda activate openms-parquet | |
| # None of the other arrow sublibraries seemed necessary, but it's an easy addition. | |
| conda install -c conda-forge libarrow libparquet -y | |
| # Get the conda environment path for later use | |
| echo "CONDA_ENV_PATH=$(conda info --base)/envs/openms-parquet" >> $GITHUB_ENV | |
| # Add conda environment to PATH for cmake to find the libraries | |
| echo "$(conda info --base)/envs/openms-parquet/bin" >> $GITHUB_PATH | |
| echo "$(conda info --base)/envs/openms-parquet/Library/bin" >> $GITHUB_PATH | |
| - name: Install Qt (Windows) | |
| if: startsWith(matrix.os, 'windows') | |
| uses: jurplel/install-qt-action@v4 | |
| with: | |
| version: '6.8.3' ## Note this version is build with win64_msvc2022_64 and should always match what we use | |
| arch: 'win64_msvc2022_64' | |
| cache: 'false' | |
| archives: 'qtsvg qtimageformats qtbase' | |
| - name: Setup build tools (and system contrib on Linux and Mac) | |
| id: tools-prefix | |
| shell: bash | |
| env: | |
| # We need to add this, otherwise brew "helpfully" updates all of the GitHub preinstalled packages, which can cause issues | |
| HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 | |
| run: | | |
| if [[ "${{ matrix.os }}" == ubuntu-* ]]; then | |
| # WARNING: Don't add any dependency install commands here. | |
| # Instead edit the following script: | |
| bash OpenMS/tools/ci/deps-ubuntu.sh | |
| echo "Testing doxygen" && doxygen --version | |
| echo "cmake_prefix=" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ matrix.os }}" == windows-* ]]; then | |
| # QT_ROOT_DIR set by Install Qt step | |
| # Add conda environment paths to CMAKE_PREFIX_PATH for finding Arrow/Parquet | |
| CONDA_CMAKE_PREFIX="$CONDA_ENV_PATH/Library;$CONDA_ENV_PATH/Library/cmake" | |
| echo "cmake_prefix=$QT_ROOT_DIR/lib/cmake;$QT_ROOT_DIR;$CONDA_CMAKE_PREFIX" >> $GITHUB_OUTPUT | |
| choco install ccache ninja -y --no-progress | |
| choco install cmake -y --no-progress | |
| ## GH CLI "SHOULD BE" installed. Sometimes I had to manually install nonetheless. Super weird. | |
| # https://github.com/actions/runner-images/blob/main/images/win/scripts/Installers/Install-GitHub-CLI.ps1 | |
| echo "C:\Program Files (x86)\GitHub CLI" >> $GITHUB_PATH | |
| # Install eigen from choco | |
| choco install eigen -y --no-progress | |
| echo "eigen_choco=C:\ProgramData\chocolatey\lib\eigen\share\cmake" >> $GITHUB_OUTPUT | |
| choco install doxygen.portable ghostscript graphviz -y --no-progress | |
| if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then | |
| # uses a custom NSIS, which provides 8-k string support and has UltraModernUI integrated already | |
| curl --no-progress-meter -L -o NSIS.tar.gz https://github.com/OpenMS/NSIS/raw/main/NSIS.tar.gz | |
| ## overwrite existing NSIS (which has only 1k-string support) | |
| 7z x -so NSIS.tar.gz | 7z x -si -ttar -aoa -o"C:/Program Files (x86)/NSIS/" | |
| fi | |
| fi | |
| if [[ "${{ matrix.os }}" == macos-* ]]; then | |
| # WARNING: Don't add any brew install commands here, edit | |
| # the following script instead: | |
| bash OpenMS/tools/ci/deps-macos.sh | |
| echo "cmake_prefix=$(brew --prefix qt)/lib/cmake;$(brew --prefix qt);$(brew --prefix libomp)" >>$GITHUB_OUTPUT | |
| fi | |
| - name: Cache contrib (Windows) | |
| if: startsWith(matrix.os, 'windows') | |
| id: cache-contrib | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ github.workspace }}/OpenMS/contrib | |
| key: ${{ runner.os }}-contrib3 | |
| - name: Download contrib build from archive (Windows) | |
| if: startsWith(matrix.os, 'windows') && steps.cache-contrib.outputs.cache-hit != 'true' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd OpenMS/contrib | |
| # Download the file using the URL fetched from GitHub | |
| gh release download -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' | |
| # Extract the archive | |
| 7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar | |
| rm contrib_build-Windows.tar.gz | |
| ls -la | |
| - name: Setup ccache cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: .ccache | |
| key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ steps.extract_branch.outputs.RUN_NAME }}-${{ github.run_number }} | |
| # Restoring: From current branch/PR, otherwise from nightly, otherwise from any branch. | |
| restore-keys: | | |
| ${{ runner.os }}-${{ runner.arch }}-ccache-${{ steps.extract_branch.outputs.RUN_NAME }} | |
| ${{ runner.os }}-${{ runner.arch }}-ccache-develop | |
| ${{ runner.os }}-${{ runner.arch }}-ccache- | |
| - name: Add THIRDPARTY | |
| shell: bash | |
| run: | | |
| # initialize THIRDPARTY | |
| cd OpenMS | |
| git submodule update --init THIRDPARTY | |
| cd .. | |
| # add third-party binaries to PATH | |
| # use flat THIRDPARTY structure | |
| THIRDPARTY_DIR="OpenMS/THIRDPARTY" | |
| mkdir -p _thirdparty | |
| # Copy the correct architecture | |
| cp -R "$THIRDPARTY_DIR/${{ steps.set-vars.outputs.tp_folder }}/$(uname -m)"/* _thirdparty/ | |
| # For MacOS 14 we also want to copy over any tools that we only have in x86_64 flavor. | |
| if [[ "${{ matrix.os }}" == macos-14 ]]; then | |
| echo "Copying x86_64 tools for MacOS 14" | |
| for tool in $THIRDPARTY_DIR/MacOS/x86_64/*; do | |
| toolname=$(basename "$tool") | |
| if [[ ! -e "_thirdparty/$toolname" ]]; then | |
| echo "Copying missing tool from x86_64: $toolname" | |
| cp -R "$tool" "_thirdparty/" | |
| fi | |
| done | |
| fi | |
| cp -R OpenMS/THIRDPARTY/All/* _thirdparty/ | |
| # add third-party binaries to PATH | |
| for thirdpartytool in '${{ github.workspace }}/_thirdparty'/* | |
| do | |
| if [[ "${{ matrix.os }}" == windows-* ]]; then | |
| # substitute slashes | |
| echo ${thirdpartytool//\//\\} >> $GITHUB_PATH | |
| else | |
| echo $thirdpartytool >> $GITHUB_PATH | |
| fi | |
| done | |
| # Upload the changelog to the same artifact that we will add the installers to. | |
| - name: Upload changelog as artifact | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: changelog.txt | |
| path: | | |
| ${{ github.workspace }}/changelog.txt | |
| - name: Build | |
| shell: bash | |
| run: | | |
| # do not fail immediately | |
| set +e | |
| # On Linux we always build pyOpenMS | |
| if [[ "${{ matrix.os }}" == ubuntu-* ]]; then | |
| export PYOPENMS=ON | |
| fi | |
| # Generate the CWL descriptors if we are going to build a package | |
| if [[ "${{steps.set-vars.outputs.pkg_type}}" != "none" ]] || | |
| [[ "${PYOPENMS:-}" == "ON" ]] | |
| then | |
| python -m venv openms-cwl | |
| source openms-cwl/bin/activate | |
| pip install -q cwltool | |
| export ENABLE_CWL=ON | |
| fi | |
| if [[ "${PYOPENMS:-}" == "ON" ]]; then | |
| python -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt | |
| fi | |
| mkdir $GITHUB_WORKSPACE/OpenMS/bld/ | |
| ${{ steps.set-vars.outputs.xvfb }} ctest --output-on-failure -V -S ../OpenMS/OpenMS/tools/ci/cibuild.cmake | |
| retVal=$? | |
| if [ $retVal -ne 0 ]; then | |
| echo -e "\033[0;31m Errors in build:" | |
| # TODO upload logs as artifact? | |
| find $GITHUB_WORKSPACE/OpenMS/bld/ -name "LastBuild*" -type f -exec cat {} \; | |
| # remove for next steps | |
| find $GITHUB_WORKSPACE/OpenMS/bld/ -name "LastBuild*" -type f -exec rm {} \; | |
| fi | |
| exit $retVal | |
| env: | |
| # TODO allow actual choice of compiler instead of using default. | |
| # Problem: not all compilers are always in the PATH. | |
| CMAKE_CXX_COMPILER: ${{ steps.set-vars.outputs.cxx_compiler }} | |
| CMAKE_PREFIX_PATH: ${{ steps.tools-prefix.outputs.cmake_prefix }} | |
| OPENMS_CONTRIB_LIBS: "${{ github.workspace }}/OpenMS/contrib" | |
| CI_PROVIDER: "GitHub-Actions" | |
| CMAKE_GENERATOR: "Ninja" | |
| SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
| BUILD_NAME: "${{ steps.extract_branch.outputs.RUN_NAME }}-${{ steps.set-vars.outputs.tp_folder }}-${{ runner.arch }}-${{ matrix.compiler }}-${{ matrix.compiler_ver }}-class-topp-${{ github.run_number }}" | |
| ENABLE_STYLE_TESTING: "OFF" | |
| ENABLE_TOPP_TESTING: "ON" | |
| ENABLE_CLASS_TESTING: "ON" | |
| ENABLE_DOCS: ${{ steps.set-vars.outputs.enable_docs }} | |
| ENABLE_GCC_WERROR: "OFF" # TODO think about that | |
| SEARCH_ENGINES_DIRECTORY: ${{ github.workspace }}/_thirdparty | |
| WITH_GUI: "ON" | |
| WITH_PARQUET: "ON" # Conda on windows, Apt on Ubuntu, Brew on MacOS | |
| SIGNING_IDENTITY: "-" | |
| ADDRESS_SANITIZER: "Off" | |
| BUILD_TYPE: "Release" | |
| OPENMP: "ON" | |
| BOOST_USE_STATIC: ${{ steps.set-vars.outputs.static_boost }} | |
| COMPILE_PXDS: ${{ startsWith(matrix.os, 'ubuntu') && github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'develop' && 'ON' || 'OFF' }} # test generation of pxds | |
| # BUILD_FLAGS: "-p:CL_MPCount=2" # For VS Generator and MSBuild | |
| BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}" # Ninja will otherwise use all cores (doesn't go well in GHA). | |
| CMAKE_CCACHE_EXE: "ccache" | |
| CCACHE_BASEDIR: ${{ github.workspace }} | |
| CCACHE_DIR: ${{ github.workspace }}/.ccache | |
| CCACHE_COMPRESS: true | |
| CCACHE_COMPRESSLEVEL: 12 | |
| CCACHE_MAXSIZE: 400M | |
| # TODO evaluate those options (we had them only on Linux before) | |
| CCACHE_SLOPPINESS: time_macros,include_file_ctime,include_file_mtime | |
| CCACHE_COMPILERCHECK: content | |
| WITH_THERMORAWFILEPARSER_TEST: "OFF" | |
| Eigen3_DIR: "${{ steps.tools-prefix.outputs.eigen_choco }}" # Eigen is installed on windows using choco | |
| USE_EXTERNAL_JSON: ${{ (startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'windows')) && 'OFF' || 'ON' }} | |
| # It's literally only Ubuntu's package manager that forcibly bundles nlohmann_json w/ arrow. The rest we can control via a cmake option in contrib + | |
| # Note that apache arrow's brew/apt packages are DIFFERENT. I believe Ubuntu's packages have Nlohmann_JSON bundled with them, while | |
| # brew doesn't. As such, this results in compatibility issues between OpenMS' Nlohmann_JSON and Ubuntu's system packages. | |
| - name: Test | |
| shell: bash | |
| run: | | |
| # On Linux we always build pyOpenMS | |
| if [[ "${{ matrix.os }}" == ubuntu-* ]]; then | |
| export PYOPENMS=ON | |
| fi | |
| ${{ steps.set-vars.outputs.xvfb }} ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake | |
| env: | |
| SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
| CI_PROVIDER: "GitHub-Actions" | |
| BUILD_NAME: "${{ steps.extract_branch.outputs.RUN_NAME }}-${{ steps.set-vars.outputs.tp_folder }}-${{ runner.arch }}-${{ matrix.compiler }}-${{ matrix.compiler_ver }}-class-topp-${{ github.run_number }}" | |
| # The rest of the vars should be saved in the CMakeCache | |
| # Load the code signing certificate and the package signing certificate TODO: Change which cert to use for non-nightly | |
| - name: Import Apple Developer Certificates | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'macos') | |
| env: | |
| APPLE_CERTIFICATE: ${{ secrets.APPLE_DEVELOPER_CERT_64 }} | |
| APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_DEVELOPMENT_PW }} | |
| INSTALLER_CERTIFICATE: ${{ secrets.APPLE_INSTALLER_CERT_64 }} | |
| INSTALLER_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_INSTALLER_PW }} | |
| KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} | |
| run: | | |
| echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12 | |
| echo $INSTALLER_CERTIFICATE | base64 --decode > installer_certificate.p12 | |
| security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain | |
| security default-keychain -s build.keychain | |
| security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain | |
| security set-keychain-settings -t 3600 -u build.keychain | |
| security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign | |
| security import installer_certificate.p12 -k build.keychain -P "$INSTALLER_CERTIFICATE_PASSWORD" -T /usr/bin/codesign | |
| security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain | |
| security find-identity -v -p codesigning build.keychain | |
| - name: Package | |
| if: steps.set-vars.outputs.pkg_type != 'none' | |
| shell: bash | |
| run: | | |
| # do not fail immediately | |
| set +e | |
| ${{ steps.set-vars.outputs.xvfb }} ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cipackage.cmake | |
| retVal=$? | |
| if [ $retVal -ne 0 ]; then | |
| echo -e "\033[0;31m Errors in packaging:" | |
| # TODO upload logs as artifact? | |
| find $GITHUB_WORKSPACE/OpenMS/bld/ -name "LastBuild*" -type f -exec cat {} \; | |
| # remove for next steps | |
| find $GITHUB_WORKSPACE/OpenMS/bld/ -name "LastBuild*" -type f -exec rm {} \; | |
| # only on win | |
| cat OpenMS/bld/_CPack_Packages/win64/NSIS/NSISOutput.log || true | |
| fi | |
| exit $retVal | |
| env: | |
| SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
| PACKAGE_TYPE: ${{ steps.set-vars.outputs.pkg_type }} | |
| SEARCH_ENGINES_DIRECTORY: "${{ github.workspace }}/_thirdparty" | |
| CI_PROVIDER: "GitHub-Actions" | |
| BUILD_NAME: "${{ steps.extract_branch.outputs.RUN_NAME }}-${{ steps.set-vars.outputs.tp_folder }}-${{ runner.arch }}-${{ matrix.compiler }}-${{ matrix.compiler_ver }}-class-topp-${{ github.run_number }}" | |
| SIGNING_IDENTITY: "Apple Development: Samuel Wein (B26WBF742H)" | |
| SIGNING_EMAIL: "[email protected]" | |
| CPACK_PRODUCTBUILD_IDENTITY_NAME: "Developer ID Installer: OpenMS Inc. (C64UCGJ5PL)" | |
| # WARNING: Here you have to make sure that only one matrix configuration per OS produces packages. See set-vars steps. | |
| - name: Upload packages as artifacts | |
| if: steps.set-vars.outputs.pkg_type != 'none' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('installer-{0}{1}', steps.set-vars.outputs.tp_folder, runner.arch == 'arm64' && '-arm64' || '') }} | |
| path: | | |
| ${{ github.workspace }}/OpenMS/bld/*.exe | |
| ${{ github.workspace }}/OpenMS/bld/*.deb | |
| ${{ github.workspace }}/OpenMS/bld/*.pkg | |
| #Zip the documentation first, since there are :: which make the upload action choke. | |
| - name: Zip Documentation | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') | |
| uses: thedoctor0/[email protected] | |
| with: | |
| type: 'zip' | |
| directory: ${{ github.workspace }}/OpenMS/bld/doc | |
| exclusions: '/CMakeFiles/* /doxygen/* /code_examples/*' | |
| filename: 'documentation.zip' | |
| # Compress the source | |
| - name: Create tarball | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| tar --exclude='bld/*' \ | |
| --exclude='OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz' \ | |
| --exclude='OpenMS/THIRDPARTY/**' \ | |
| --exclude='.git/**' \ | |
| --exclude='.github/**' \ | |
| --exclude='vcpkg*/**' \ | |
| -czf OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz -C ${{ github.workspace }} OpenMS | |
| # Upload the source tar | |
| - name: Upload source tar as artifact | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz | |
| path: | | |
| ${{ github.workspace }}/OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz | |
| # Only upload docs when we are building the package, use the ubuntu build simply 'cause its fast | |
| - name: Upload Documentation as artifacts | |
| if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: | | |
| ${{ github.workspace }}/OpenMS/bld/doc/documentation.zip | |
| - name: Generate KNIME descriptors and payloads | |
| # We never want to build the KNIME update site on our second Ubuntu + Clang matrix entry even if inputs.knime is true, so we check for that specific os + compiler combo | |
| if: steps.set-vars.outputs.pkg_type != 'none' || ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && startsWith(matrix.compiler, 'clang++') ) ) | |
| shell: bash | |
| run: | | |
| cd $GITHUB_WORKSPACE/OpenMS/bld/ | |
| # TODO use CTest or script to upload to CDash? | |
| cmake -DSEARCH_ENGINES_DIRECTORY="$GITHUB_WORKSPACE/_thirdparty" -DENABLE_PREPARE_KNIME_PACKAGE=ON . | |
| cmake --build . --target prepare_knime_package | |
| - name: Upload KNIME payload and descriptors as artifacts | |
| # We never want to build the KNIME update site on our second Ubuntu + Clang matrix entry even if inputs.knime is true, so we check for that specific os + compiler combo | |
| if: steps.set-vars.outputs.pkg_type != 'none' || ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && startsWith(matrix.compiler, 'clang++') ) ) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('knime-{0}{1}', steps.set-vars.outputs.tp_folder, runner.arch == 'arm64' && '-arm64' || '') }} | |
| path: ${{ github.workspace }}/OpenMS/bld/knime/**/* | |
| deploy-installer: | |
| if: github.ref == 'refs/heads/nightly' || contains(github.ref, 'release/') || inputs.package | |
| runs-on: ubuntu-latest | |
| needs: build-and-test | |
| outputs: | |
| release_text_contents: ${{ steps.mk_release_txt.outputs.release_text_contents }} | |
| steps: | |
| - name: Extract branch/PR infos | |
| shell: bash | |
| run: echo "RUN_NAME=${{ github.event.pull_request && github.event.number || github.ref_name }}" >> $GITHUB_ENV | |
| - name: Download installer artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: installer-* | |
| merge-multiple: true | |
| - name: Download source archive as artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: OpenMS-${{ needs.build-and-test.outputs.version_number }}.tar.gz | |
| - name: Download changelog as artifact | |
| if: inputs.do_release | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: changelog.txt | |
| - name: Upload installer | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| run: | | |
| echo "Upload" | |
| if [[ "${{ github.ref_name }}" == "nightly" ]]; then | |
| folder=nightly | |
| elif [[ "${{ github.ref_name }}" == release/* ]]; then | |
| folder=${{ github.ref_name }} | |
| else | |
| folder=experimental/${{ github.ref_name }} | |
| fi | |
| echo "Uploading installers to: " $folder | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/* "$USER@$HOST:/OpenMSInstaller/${folder}" | |
| - name: Make installer latest | |
| if: inputs.mark_as_latest | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| run: | | |
| if [[ "${{ github.ref_name }}" == release/* ]]; then | |
| folder=${{ github.ref_name }} | |
| else | |
| echo refusing to mark non-release branch as latest | |
| exit 1 | |
| fi | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| ln -s ../$folder latest #create link to the release folder | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" latest "$USER@$HOST:/OpenMSInstaller/release" | |
| - name: create RELEASE_TEXT | |
| id: mk_release_txt | |
| if: inputs.do_release | |
| shell: bash | |
| run: | | |
| #TODO Handle changelog_params | |
| ## Header | |
| cat > releaseTextHeader.txt << EOF | |
| Dear OpenMS-Users, | |
| we are proud to announce the release of OpenMS ${{ needs.build-and-test.outputs.version_number }}. Grab it <a href="https://abibuilder.cs.uni-tuebingen.de/archive/openms/OpenMSInstaller/release/${{ needs.build-and-test.outputs.version_number }}">here</a> | |
| In the following you find all important changes to the previous version: | |
| EOF | |
| ## Header | |
| cat > releaseTextParamHeader.txt << EOF | |
| Additionally, the following changes were made to parameters of our TOPP tools: | |
| EOF | |
| ##Footer | |
| cat > releaseTextFooter.txt << EOF | |
| Best regards, | |
| The OpenMS-Developers | |
| EOF | |
| RELEASE_TEXT=$(cat releaseTextHeader.txt $GITHUB_WORKSPACE/changelog.txt releaseTextFooter.txt) | |
| RELEASE_TEXT_ESCAPED_QUOTES="${RELEASE_TEXT//\"/\\\"}" | |
| cat releaseTextHeader.txt $GITHUB_WORKSPACE/changelog.txt releaseTextFooter.txt > RELEASE_TEXT_GH.md | |
| echo release_text_contents=`base64 -w0 RELEASE_TEXT_GH.md` >> $GITHUB_OUTPUT | |
| - name: run GHA release action. | |
| id: create_release | |
| if: inputs.do_release | |
| uses: ncipollo/[email protected] | |
| with: | |
| bodyFile: RELEASE_TEXT_GH.md | |
| tag: ${{ github.ref_name }} | |
| draft: true | |
| artifactErrorsFailBuild: true | |
| makeLatest: ${{ inputs.mark_as_latest }} | |
| artifacts: | |
| ${{ github.workspace }}/*.exe, ${{ github.workspace }}/*.deb, ${{ github.workspace }}/*.pkg, ${{ github.workspace }}/*.tar.gz | |
| deploy-docs: | |
| if: github.ref == 'refs/heads/nightly' || contains(github.ref, 'release/') || inputs.package | |
| runs-on: ubuntu-latest | |
| needs: build-and-test | |
| steps: | |
| - name: Extract branch/PR infos | |
| shell: bash | |
| run: echo "RUN_NAME=${{ github.event.pull_request && github.event.number || github.ref_name }}" >> $GITHUB_ENV | |
| - name: Download Documentation | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: documentation | |
| path: docs | |
| - name: Upload Documentation | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| run: | | |
| unzip $GITHUB_WORKSPACE/docs/documentation.zip -d $GITHUB_WORKSPACE/docs | |
| rm $GITHUB_WORKSPACE/docs/documentation.zip | |
| echo "Upload" | |
| if [[ "${{ github.ref_name }}" == "nightly" ]]; then | |
| folder=nightly | |
| elif [[ "${{ github.ref_name }}" == release/* ]]; then | |
| folder=${{ github.ref_name }} | |
| else | |
| folder=experimental/${{ github.ref_name }} | |
| fi | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| # Upload documentation | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/docs/ "$USER@$HOST:/Documentation/${folder}" | |
| - name: Make documentation latest | |
| if: inputs.mark_as_latest | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| if [[ "${GITHUB_REF_NAME}" == release/* ]]; then | |
| # Remove 'release/' prefix using parameter expansion | |
| folder=${GITHUB_REF_NAME#release/} | |
| else | |
| echo refusing to mark non-release branch as latest | |
| exit 1 | |
| fi | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| ln -s ./$folder latest #we can use the same link from above. | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release" | |
| # TODO create softlinks to latest nightly | |
| # TODO create and upload file hashes, at least for release candidate | |
| build-deploy-knime-updatesite: | |
| env: | |
| KNIME: 5.3 | |
| JAVA_VER: 17 | |
| PLUGIN_BUILD: ${{ github.workspace }}/plugin-build | |
| PLUGIN_SOURCE: ${{ github.workspace }}/plugin-source | |
| if: github.ref == 'refs/heads/nightly' || contains(github.ref, 'release/') || inputs.knime | |
| runs-on: ubuntu-latest | |
| needs: build-and-test | |
| steps: | |
| - name: Extract branch/PR infos | |
| shell: bash | |
| run: echo "RUN_NAME=${{ github.event.pull_request && github.event.number || github.ref_name }}" >> $GITHUB_ENV | |
| - name: Download KNIME artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: knime-* | |
| merge-multiple: true | |
| path: ${{ env.PLUGIN_SOURCE }} | |
| # Should be the default. | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.JAVA_VER }} | |
| distribution: 'temurin' | |
| - name: Generate KNIME plugin sources | |
| shell: bash | |
| run: | | |
| ## Setup functions | |
| function replace_qualifier { | |
| find $1 -name MANIFEST.MF -exec sed -i -e "s/qualifier/$2/g" {} \; | |
| } | |
| ## replace only first occurence of SNAPSHOT (second is the version of the parent pom). | |
| function replace_snapshot { | |
| find $1 -name pom.xml -exec sed -i -e "1 h;1! H;\$! d;$ {g;s/-SNAPSHOT/\.$2/;}" {} \; | |
| } | |
| ## Setup variables | |
| ## GKN git clone (see SCM subfolder settings above) | |
| GKN_PLUGIN_PATH=${GITHUB_WORKSPACE}/GenericKnimeNodes | |
| git clone -q https://github.com/genericworkflownodes/GenericKnimeNodes/ --branch develop --single-branch $GKN_PLUGIN_PATH | |
| ## Create folders | |
| mkdir ${PLUGIN_BUILD} | |
| PLUGIN_SOURCE_CONTRIB_PLUGINS=${PLUGIN_SOURCE}/de.openms.feature/contributing-plugins | |
| mkdir -p ${PLUGIN_SOURCE_CONTRIB_PLUGINS} | |
| PLUGIN_SOURCE_THIRDPARTY_CONTRIB_PLUGINS=${PLUGIN_SOURCE}/de.openms.thirdparty.feature/contributing-plugins | |
| mkdir -p ${PLUGIN_SOURCE_THIRDPARTY_CONTRIB_PLUGINS} | |
| ## We check exemplarily in de.openms.lib. It should never happen that one platform produced one plugin but not the other. | |
| pushd ${PLUGIN_SOURCE}/de.openms.feature/de.openms.lib/ | |
| amount=$(ls -1 payload | wc -l) | |
| if [ $amount -le 2 ]; then | |
| echo "Not enough payloads for each platform. Aborting..." | |
| exit 1 | |
| fi | |
| popd | |
| ## Clone the contributing plugins into the plugin source folder and update their "last modified" date to | |
| ## the last git commit | |
| pushd ${PLUGIN_SOURCE_CONTRIB_PLUGINS} | |
| git clone -q https://github.com/OpenMS/de.openms.knime --branch develop --single-branch . | |
| CONTRIBUTING_DATE=$(git log -1 --format="%cd" --date="format:%Y%m%d%H%M") | |
| echo "Setting qualifier/SNAPSHOT for de.openms.knime contrib plugins to ${CONTRIBUTING_DATE}!" | |
| git log -1 --format="%cd" --date=iso | |
| # TODO just iterate over all folders? | |
| replace_qualifier de.openms.knime.startupCheck ${CONTRIBUTING_DATE} | |
| replace_qualifier de.openms.knime.mzTab ${CONTRIBUTING_DATE} | |
| replace_qualifier de.openms.knime.textexporter_reader ${CONTRIBUTING_DATE} | |
| replace_qualifier de.openms.knime.importers ${CONTRIBUTING_DATE} | |
| replace_qualifier de.openms.knime.qchandling ${CONTRIBUTING_DATE} | |
| replace_qualifier de.openms.thirdparty.knime.startupCheck ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.startupCheck ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.mzTab ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.textexporter_reader ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.importers ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.qchandling ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.thirdparty.knime.startupCheck ${CONTRIBUTING_DATE} | |
| # move to thirdparty feature so they get installed together | |
| mv de.openms.thirdparty.knime.startupCheck ${PLUGIN_SOURCE_THIRDPARTY_CONTRIB_PLUGINS}/de.openms.thirdparty.knime.startupCheck | |
| rm -r .git | |
| rm -r de.openms.knime.startupCheck.test | |
| popd | |
| ## New contrib plugin: de.openms.knime.dynamicJSViewers | |
| git clone -q https://github.com/genericworkflownodes/de.openms.knime.dynamicJSViewers --branch master --single-branch | |
| pushd de.openms.knime.dynamicJSViewers | |
| CONTRIBUTING_DATE=$(git log -1 --format="%cd" --date="format:%Y%m%d%H%M") | |
| popd | |
| mv de.openms.knime.dynamicJSViewers/de.openms.knime.dynamicJSViewers.feature ${PLUGIN_SOURCE}/ | |
| mkdir ${PLUGIN_SOURCE}/de.openms.knime.dynamicJSViewers.feature/contributing-plugins | |
| rm -rf ${PLUGIN_SOURCE}/de.openms.knime.dynamicJSViewers.feature/.settings | |
| mv de.openms.knime.dynamicJSViewers/de.openms.knime.dynamicJSViewers ${PLUGIN_SOURCE}/de.openms.knime.dynamicJSViewers.feature/contributing-plugins/ | |
| pushd ${PLUGIN_SOURCE}/de.openms.knime.dynamicJSViewers.feature/contributing-plugins/ | |
| echo "Setting qualifier/SNAPSHOT for de.openms.knime.dynamicJSViewers to ${CONTRIBUTING_DATE}" | |
| replace_qualifier de.openms.knime.dynamicJSViewers ${CONTRIBUTING_DATE} | |
| replace_snapshot de.openms.knime.dynamicJSViewers ${CONTRIBUTING_DATE} | |
| popd | |
| rm -rf de.openms.knime.dynamicJSViewers | |
| # build the plugin source code | |
| cd ${GKN_PLUGIN_PATH} | |
| ant -Dplugin.dir=${PLUGIN_SOURCE} -Dgenerate.extra.arguments="-r -u" -Dcustom.plugin.generator.target=${PLUGIN_BUILD} | |
| - name: Build KNIME update site | |
| shell: bash | |
| run: | | |
| # fix folder structure | |
| mv ${PLUGIN_BUILD}/.mvn ./ | |
| mv ${PLUGIN_BUILD}/* ./ | |
| git clone --depth 1 --branch master https://github.com/genericworkflownodes/buildresources | |
| # a special class that maps node names from old openms to openms thirdparty (we should be able to remove it future releases) | |
| cp buildresources/de.openms.thirdparty/src/de/openms/thirdparty/knime/OpenMSNodeFactoryClassMapper.java de.openms.thirdparty/src/de/openms/thirdparty/knime/ | |
| # insert contents of the snippet before </plugin> in the plugin.xml. The snippet registers the class mapper. | |
| cat buildresources/de.openms.thirdparty/plugin.xml.snippet | |
| f1="$(cat buildresources/de.openms.thirdparty/plugin.xml.snippet)" | |
| awk -vf1="$f1" '/<\/plugin>/{print f1;print;next}1' de.openms.thirdparty/plugin.xml > buildresources/my.tmp && mv buildresources/my.tmp de.openms.thirdparty/plugin.xml | |
| rm -r buildresources | |
| rm -r plugin-build | |
| # remove maven pom.xmls since we will do a full pomless build. poms are only necessary if you want to develop on a single plugin, e.g., in eclipse. | |
| find . -mindepth 2 -name "pom.xml*" -exec rm {} \; | |
| # Build update site. Downloads necessary plugins from the dependency update sites from KNIME and GKN (for the correct KNIME version) | |
| mvn --no-transfer-progress "-Dknime.version=${KNIME}" "-Djava.majorversion=${JAVA_VER}" "-Dgkn.update.site=https://abibuilder.cs.uni-tuebingen.de/archive/gkn/updateSite/${KNIME}/" clean verify | |
| - name: Upload KNIME update site | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| run: | | |
| echo "Upload" | |
| if [[ "${{ github.ref_name }}" == "nightly" ]]; then | |
| folder=nightly | |
| elif [[ "${{ github.ref_name }}" == release/* ]]; then | |
| folder=${{ github.ref_name }} | |
| else | |
| folder=experimental/${{ github.ref_name }} | |
| fi | |
| echo "Uploading KNIME site to:" $folder | |
| # correct permissions of files to upload | |
| chmod -R o+r $GITHUB_WORKSPACE/de.openms.update/target/repository/* | |
| chmod o+x $GITHUB_WORKSPACE/de.openms.update/target/repository/features | |
| chmod o+x $GITHUB_WORKSPACE/de.openms.update/target/repository/plugins | |
| #Load private key into file | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/de.openms.update/target/repository/* "$USER@$HOST:/knime-plugin/updateSite/$folder/" | |
| - name: Make KNIME installer latest | |
| if: inputs.mark_as_latest | |
| shell: bash | |
| env: | |
| PASS: ${{ secrets.ARCHIVE_RRSYNC_SSH }} | |
| USER: ${{ secrets.ARCHIVE_RRSYNC_USER }} | |
| HOST: ${{ secrets.ARCHIVE_RRSYNC_HOST }} | |
| PORT: ${{ secrets.ARCHIVE_RRSYNC_PORT }} | |
| run: | | |
| echo "Upload" | |
| if [[ "${{ github.ref_name }}" == release/* ]]; then | |
| folder=${{ github.ref_name }} | |
| else | |
| echo refusing to mark non-release branch as latest | |
| exit 1 | |
| fi | |
| mkdir -p ~/.ssh/ | |
| echo "$PASS" > ~/.ssh/private.key | |
| sudo chmod 600 ~/.ssh/private.key | |
| ln -s ../$folder latest #create link to the release folder | |
| rsync --progress -avz -e "ssh -i ~/.ssh/private.key -p $PORT -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release" | |
| do-release: | |
| if: inputs.do_release | |
| runs-on: ubuntu-latest | |
| needs: [build-deploy-knime-updatesite,deploy-installer, build-and-test] | |
| permissions: write-all | |
| steps: | |
| - name: get a token so that we can update the tags | |
| uses: actions/create-github-app-token@v1 | |
| id: app-token | |
| with: | |
| app-id: ${{ vars.OPENMS_GITHUB_APP_ID }} | |
| private-key: ${{ secrets.OPENMS_GITHUB_APP_PRIVATE_KEY }} | |
| owner: ${{ github.repository_owner }} | |
| - name: Extract branch/PR infos | |
| shell: bash | |
| run: echo "RUN_NAME=${{ github.event.pull_request && github.event.number || github.ref_name }}" >> $GITHUB_ENV | |
| # We created the draft release during deploy-installer step. Now we want to publish it. | |
| - name: Publish OpenMS release | |
| id: publish_release | |
| if: inputs.do_release | |
| uses: ncipollo/[email protected] | |
| with: | |
| tag: ${{ github.ref_name }} | |
| draft: false | |
| allowUpdates: true | |
| artifactErrorsFailBuild: true | |
| makeLatest: ${{ inputs.mark_as_latest }} | |
| omitBodyDuringUpdate: true | |
| omitNameDuringUpdate: true | |
| updateOnlyUnreleased: true | |
| # NB we create the tag for the OpenMS repo next in a separate action. | |
| # SPW TODO: Move the script here to a file once its stable | |
| - id: bash_create_tags | |
| name: create tags for other repos | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| function handleGitTag() { | |
| REPO=$1 | |
| SHA=$2 | |
| TAG_NAME="${{ env.RUN_NAME }}" | |
| # Check if the tag exists | |
| TAG_EXISTS=$(gh api \ | |
| --method GET \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${REPO}/git/refs/tags/${TAG_NAME} 2>/dev/null || echo "TAG_NOT_FOUND") | |
| if [[ "$TAG_EXISTS" == "TAG_NOT_FOUND" ]]; then | |
| # Tag doesn't exist, create it | |
| echo "Creating tag ${TAG_NAME} in ${REPO} pointing to ${SHA}" | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${REPO}/git/refs \ | |
| -f ref="refs/tags/${TAG_NAME}" \ | |
| -f sha="${SHA}" | |
| else | |
| # Tag exists, check if it points to the same SHA | |
| EXISTING_SHA=$(echo $TAG_EXISTS | jq -r '.object.sha') | |
| if [[ "$EXISTING_SHA" == "$SHA" ]]; then | |
| # Tag already points to the correct SHA, do nothing | |
| echo "Tag ${TAG_NAME} already exists in ${REPO} and points to the correct SHA: ${SHA}" | |
| else | |
| # Tag exists but points to a different SHA, throw an error | |
| echo "Error: Tag ${TAG_NAME} already exists in ${REPO} but points to ${EXISTING_SHA} instead of ${SHA}. Manual intervention required." | |
| exit 1 | |
| fi | |
| fi | |
| } | |
| DEOPENMS_SHA=$(curl -s -X GET https://api.github.com/repos/OpenMS/de.openms.knime/git/ref/heads/develop |jq -r '.object.sha') | |
| GKN_SHA=$(curl -s -X GET https://api.github.com/repos/genericworkflownodes/GenericKnimeNodes/git/ref/heads/develop |jq -r '.object.sha') | |
| JSViewer_SHA=$(curl -s -X GET https://api.github.com/repos/genericworkflownodes/de.openms.knime.dynamicJSViewers/git/ref/heads/master |jq -r '.object.sha') | |
| CONTRIB_SHA=$(curl -s -X GET https://api.github.com/repos/OpenMS/OpenMS/contents/contrib\?ref\=${{ github.sha }} | jq -r ".sha") | |
| THIRDPARTY_SHA=$(curl -s -X GET https://api.github.com/repos/OpenMS/OpenMS/contents/THIRDPARTY\?ref\=${{ github.sha }} | jq -r ".sha") | |
| PYDOCS_SHA=$(curl -s -X GET https://api.github.com/repos/OpenMS/OpenMS/contents/src/pyOpenMS/pyopenms-docs\?ref\=${{ github.sha }} | jq -r ".sha") | |
| TUTORIAL_SHA=$(curl https://api.github.com/repos/OpenMS/Tutorials/git/refs/heads/master | jq -r ".object.sha") | |
| DOCS_SHA=$(curl https://api.github.com/repos/OpenMS/OpenMS-docs/git/refs/heads/develop | jq -r ".object.sha") | |
| handleGitTag OpenMS/contrib $CONTRIB_SHA | |
| handleGitTag OpenMS/pyopenms-docs $PYDOCS_SHA | |
| handleGitTag OpenMS/THIRDPARTY $THIRDPARTY_SHA | |
| handleGitTag OpenMS/Tutorials $TUTORIAL_SHA | |
| handleGitTag OpenMS/OpenMS-docs $DOCS_SHA | |
| handleGitTag OpenMS/de.openms.knime $DEOPENMS_SHA | |
| # Uncomment when permissions are fixed | |
| # handleGitTag genericworkflownodes/de.openms.knime.dynamicJSViewers $JSViewer_SHA | |
| - name: Merge to Develop | |
| if: inputs.mark_as_latest | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { repo, owner } = context.repo; | |
| const result = await github.rest.pulls.create({ | |
| title: 'Merge latest release into develop', | |
| owner, | |
| repo, | |
| head: '${{ github.ref_name }}', | |
| base: 'develop', | |
| body: [ | |
| 'This PR is auto-generated by', | |
| '[actions/github-script](https://github.com/actions/github-script).' | |
| ].join('\n') | |
| }); | |
| github.rest.issues.addLabels({ | |
| owner, | |
| repo, | |
| issue_number: result.data.number, | |
| labels: ['feature', 'automated pr'] | |
| }); | |
| create-website-update: | |
| if: inputs.announce_release | |
| runs-on: ubuntu-latest | |
| needs: [deploy-installer, build-and-test] | |
| permissions: write-all | |
| steps: | |
| - name: get a token so that we can update the website | |
| uses: actions/create-github-app-token@v1 | |
| id: app-token | |
| with: | |
| app-id: ${{ vars.OPENMS_GITHUB_APP_ID }} | |
| private-key: ${{ secrets.OPENMS_GITHUB_APP_PRIVATE_KEY }} | |
| owner: ${{ github.repository_owner }} | |
| - name: checkout website | |
| uses: actions/checkout@v4 | |
| with: | |
| path: OpenMS-website | |
| repository: OpenMS/OpenMS-website | |
| token: ${{ steps.app-token.outputs.token }} | |
| - name: create text for website and update it. | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| echo '${{ needs.deploy-installer.outputs.release_text_contents }}' | base64 -d >> OpenMS-website/content/en/news/release${{ needs.build-and-test.outputs.version_number }}.md | |
| pushd OpenMS-website | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| git checkout -b "releaseannouncement${{ needs.build-and-test.outputs.version_number }}" | |
| git add content/en/news/release${{ needs.build-and-test.outputs.version_number }}.md | |
| git commit -am "Announce release ${{ needs.build-and-test.outputs.version_number }}" | |
| git push --set-upstream origin releaseannouncement${{ needs.build-and-test.outputs.version_number }} | |
| gh pr create --title "Release announcement for ${{ needs.build-and-test.outputs.version_number }}" --body "announces ${{ needs.build-and-test.outputs.version_number }}" | |
| popd |