Add automatic build of html documentation.

.github/workflows/docs.yml

@@ -0,0 +1,367 @@
+#  manuals -> main latest e LTS
+#  dev/<branch>/arch
+#
+#  link a pdf e constraint coi test ok
+
+
+# Tree Structure
+# --------------
+#
+# .oq-engine-new
+#     <x.y>|main
+#         PDF
+#             <pdf_docs>
+#         reference
+#         manual
+#         advanced
+#     latest -> <latest x.y>
+#     LTS    -> <LTS x.y>
+#     reference
+#         latest -> ../latest/reference
+#         LTS    -> ../LTS/reference
+#         main -> ../main/reference
+#     manual
+#         latest -> ../latest/manual
+#         LTS    -> ../LTS/manual
+#         main -> ../main/manual
+#     advanced
+#         latest -> ../latest/advanced
+#         LTS    -> ../LTS/advanced
+#         main -> ../main/advanced
+# manuals
+#     OpenQuake Manual <x.y>.pdf -> ../.oq-engine-new/<x.y>/PDF/OpenQuake Manual <x.y>.pdf
+#     OpenQuake Manual latest.pdf -> OpenQuake Manual <latest_(x.y)>.pdf
+#     OpenQuake Manual LTS.pdf -> OpenQuake Manual <LTS_(x.y)>.pdf
+#
+# .dev
+#     .oq-engine-new
+#         <branch>
+#             PDF
+#                 <pdf_docs>
+#             reference
+#             manual
+#             advanced
+#     manuals
+#         OpenQuake Manual <branch>.pdf -> ../.oq-engine-new/<branch>/PDF/OpenQuake Manual <branch>.pdf
+#
+---
+name: Docs
+on:
+  workflow_dispatch:
+    inputs:
+      oq-release:
+        description: 'OQ Release'
+        required: true
+        default: 'dev'
+        type: choice
+        options:
+          - dev
+          - main
+#          - release
+#          - LTS
+  push:
+    branches: [main, 'engine-*', moster, 'ongine-*']
+
+  pull_request:
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    env:
+      EV_NAME: ${{ github.event_name }}
+      # during devel was DOCS_BASE: ".oq-engine-docs-refact/"
+      DOCS_BASE: ""
+      GITHUB_REF: ${{ github.ref }}
+      GITHUB_HD_REF: ${{ github.head_ref }}
+      GITHUB_BS_REF: ${{ github.base_ref }}
+
+    steps:
+      - name: run if workflow_dispatch
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        run: echo "Action triggered by 'workflow_dispatch' ($EV_NAME)"
+      - name: run if push
+        if: github.event_name == 'push'
+        shell: bash
+        run: echo "Action triggered by 'push' ($EV_NAME)"
+      - name: identify github reference
+        run: |
+          echo "DOCS_BASE: [$DOCS_BASE]"
+          echo "REF: $GITHUB_REF"
+          echo "REF_NAME: $GITHUB_REF_NAME"
+          echo "HD_REF: $GITHUB_HD_REF"
+          echo "BS_REF: $GITHUB_BS_REF"
+      - name: Check out the codebase
+        uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Upgrade pip and install requirements
+        run: |
+          pip install -U pip
+          pip install -r ./docs/requirements.txt
+#      - name: Install oq engine
+#        run: |
+#          pip install -e .[dev]
+#          python3 -c "import openquake.baselib as m; print(m.__version__.rpartition('.')[0])"
+      - name: Make docs and rsync to docs.openquake.org
+        shell: bash
+        env:
+          # Used
+          BUILD: ${{ github.event.inputs.oq-release }}
+          DOCS_SSH: ${{ secrets.DOCS_ARTIFACTS }}
+          # GITHUB_PULL_REQUEST: ${{ github.event.number }}
+          GITHUB_DEF_BR: ${{ github.event.repository.default_branch }}
+          GITHUB_REF: ${{ github.ref }}
+          GITHUB_HD_REF: ${{ github.head_ref }}
+          GITHUB_BS_REF: ${{ github.base_ref }}
+          UPLOAD: ${{ github.event.inputs.git-ref }}
+
+          # JUST TO TEST MULTI_DOCS
+          MULTI_DOCS_FOR_DEVEL: "true"
+        run: |
+          bash --version
+          #          LTS_VER=$(grep '^Current LTS' README.md  | sed 's/.*OpenQuake Engine //g;s/\*\*.*//g')
+          LTS_VER="TODO"
+          # ENG_VER=$(python3 -c "import openquake.baselib as m; print(m.__version__.rpartition('.')[0])")
+          ENG_VER="TODO"
+          if [ -z "$BUILD" ]; then
+              if [[ "$GITHUB_REF_NAME" == "main" || "$GITHUB_REF_NAME" == "moster" ]]; then
+                  BUILD=main
+              elif [[ "$GITHUB_REF_NAME" =~ ^engine- || "$GITHUB_REF_NAME" =~ ^ongine- ]]; then
+                  BRANCH_VER="${GITHUB_REF_NAME#engine-}"
+                  if [ "$BRANCH_VER" == "$GITHUB_REF_NAME" ]; then
+                      BRANCH_VER="${GITHUB_REF_NAME#ongine-}"
+                  fi
+
+                  # TEST: ENABLE IF NEEDED
+                  # ENG_VER=$BRANCH_VER
+                  if [ "$BRANCH_VER" != "$ENG_VER" ]; then
+                      echo "Versions don't match (BRANCH_VER: [$BRANCH_VER] ENG_VER: [$ENG_VER])"
+                      exit 2
+                  fi
+                  if [[ "$BRANCH_VER" == "$LTS_VER" ]]; then
+                      BUILD=LTS
+                  else
+                      BUILD=release
+                  fi
+              else
+                  BUILD=dev
+              fi
+          fi
+
+          # REL_VERS="$(git ls-remote --heads | grep -v '^From' | grep '/engine-' | sed 's/^.*ngine-//g' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n)"
+          REL_VERS="TODO"
+          LATEST_VER="$(echo "$REL_VERS" | tail -n 1)"
+          # ALL_VERS="$((git ls-remote --heads | grep -v '^From' | grep '/engine-' | sed 's/^.*ngine-//g' ; echo "$ENG_VER") | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | uniq)"
+          ALL_VERS="TODO"
+          PDFDOCS="manuals/"
+          TARGET_LINK=""
+          PDF_VER_LINK=""
+          if [ "$BUILD" == "release" -o "$BUILD" == "LTS" ]; then
+              TARGET=".oq-engine-new/$ENG_VER/"
+              # TARGET_LINK="${DOCS_BASE}.oq-engine-new/$BUILD"
+              PDF_VER="${ENG_VER}"
+              if [ "$BUILD" == "release" ]; then
+                  if [ "$ENG_VER" == "$LATEST_VER" ]; then
+                      PDF_VER_LINK="latest"
+                  fi
+              elif [ "$BUILD" == "LTS" ]; then
+                  PDF_VER_LINK="LTS"
+              fi
+          elif [ "$BUILD" == "main" ]; then
+              TARGET=".oq-engine-new/main/"
+              PDF_VER="(main)"
+          elif [ "$BUILD" == "dev" ]; then
+              DOCS_BASE="${DOCS_BASE}.dev/"
+              if echo "$GITHUB_REF_NAME" | grep -q '[0-9]\+/merge'; then
+                  TARGET=".oq-engine-new/$GITHUB_HD_REF/"
+                  PDF_VER="($GITHUB_HD_REF)"
+              else
+                  TARGET=".oq-engine-new/$GITHUB_REF_NAME/"
+                  PDF_VER="($GITHUB_REF_NAME)"
+              fi
+              PDFDOCS="manuals/"
+          else
+              echo "BUILD [$BUILD] not recognized"
+              exit 3
+          fi
+          PDFDEST=${PDFDOCS}
+          # TEST PER PDF_VER_LINK
+          # PDF_VER_LINK=test_ver_link
+
+          echo "LATEST_VER:          [$LATEST_VER]"
+          echo "BUILD:               [$BUILD]"
+          echo "LTS_VER:             [$LTS_VER]"
+          echo "ENG_VER:             [$ENG_VER]"
+          echo "TARGET:              [$TARGET]"
+          echo "TARGET_LINK:         [$TARGET_LINK]"
+          echo "PDF_VER:             [$PDF_VER]"
+          echo "PDF_VER_LINK:        [$PDF_VER_LINK]"
+          echo "PDFDOCS              [$PDFDOCS]"
+          echo "PDFDEST              [$PDFDEST]"
+          echo "----"
+          echo "GITHUB_PULL_REQUEST: [$GITHUB_PULL_REQUEST]"
+          echo "GITHUB_DEF_BR:       [$GITHUB_DEF_BR]"
+          echo "GITHUB_REF:          [$GITHUB_REF]"
+          echo "GITHUB_HD_REF:       [$GITHUB_HD_REF]"
+          echo "GITHUB_BS_REF:       [$GITHUB_BS_REF]"
+          echo "UPLOAD:              [$UPLOAD]"
+
+          # multi-version doc switcher could be runned just from main because
+          # is the only one known the main version (from sources)
+          # set -x
+          # if [ "$MULTI_DOCS_FOR_DEVEL" == "true" -o  "$BUILD" == "main" ]; then
+          #     ddown_adv="["
+          #     ddown_man="["
+          #     for v in $ALL_VERS; do
+          #         ver_maj="$(echo "$v" | sed 's/\([0-9]\+\)\..*/\1/g')"
+          #         ver_min="$(echo "$v" | sed 's/[0-9]\+\.\(.*\)$/\1/g')"
+
+          #         pfx=""
+          #         sfx=""
+          #         if [ "$v" = "$ENG_VER" ]; then
+          #             pfx="development"
+          #             main_rename="true"
+          #         else
+          #             main_rename="false"
+          #         fi
+          #         # if [ "$v" = "$LATEST_VER" ]; then
+          #         #     if [ "$sfx" ]; then
+          #         #         sfx="${sfx}, latest"
+          #         #    else
+          #         #         sfx="latest"
+          #         #     fi
+          #         # fi
+          #         if [ "$v" = "$LTS_VER" ]; then
+          #             if [ "$sfx" ]; then
+          #                 sfx="${sfx}, LTS"
+          #             else
+          #                 sfx="LTS"
+          #             fi
+          #         fi
+          #         if [ "$sfx" ]; then
+          #             sfx=" (${sfx})"
+          #         fi
+
+          #         # policies for the user manual version switcher menu
+          #         if ( (( "$ver_maj" == 3  )) && (( "$ver_min" >= 16 )) ) || (( "$ver_maj" > 3 )); then
+          #             if [ "$main_rename" == "true" ]; then
+          #                 ver_name="${pfx}"
+          #                 ver_val="main"
+          #             else
+          #                 ver_name="${ver_maj}.${ver_min}${sfx}"
+          #                 ver_val="${ver_maj}.${ver_min}"
+          #             fi
+          #             ver_url="https://docs.openquake.org/${DOCS_BASE}.oq-engine-new/${ver_val}/manual/"
+          #             if [ "$ddown_man" != "[" ]; then
+          #                ddown_man="${ddown_man}, "
+          #             fi
+          #             ddown_man="${ddown_man}{ \"name\": \"${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }"
+          #         fi
+
+          #         # policies for the advanced manual version switcher menu
+          #         if ( (( "$ver_maj" == 3  )) && (( "$ver_min" >= 16 )) ) || (( "$ver_maj" > 3 )); then
+          #             if [ "$main_rename" == "true" ]; then
+          #                 ver_name="${pfx}"
+          #                 ver_val="main"
+          #             else
+          #                 ver_name="${ver_maj}.${ver_min}${sfx}"
+          #                 ver_val="${ver_maj}.${ver_min}"
+          #             fi
+          #             ver_url="https://docs.openquake.org/${DOCS_BASE}.oq-engine-new/${ver_val}/advanced/"
+          #             if [ "$ddown_adv" != "[" ]; then
+          #                ddown_adv="${ddown_adv}, "
+          #             fi
+          #             ddown_adv="${ddown_adv}{ \"name\": \"${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }"
+          #         fi
+
+          #     done
+          #     ddown_man="${ddown_man}]"
+          #     echo "$ddown_man" > dot_ddown_man.json
+
+          #     ddown_adv="${ddown_adv}]"
+          #     echo "$ddown_adv" > dot_ddown_adv.json
+          # fi
+          set -x
+          gpg --version
+          gpg --quiet --batch --yes --decrypt --passphrase="$DOCS_SSH" --output ./.deploy_rsa ./.deploy_docs.enc
+          chmod 600 ./.deploy_rsa
+          eval $(ssh-agent -s) && ssh-add ./.deploy_rsa
+
+          # if [ -f "dot_ddown_man.json" ]; then
+          #     rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' --rsync-path="mkdir -p \"${DOCS_BASE}.oq-engine-new\" && rsync" dot_ddown_man.json "[email protected]:${DOCS_BASE}.oq-engine-new/.ddown_man.json"
+          # fi
+          # if [ -f "dot_ddown_adv.json" ]; then
+          #     rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' --rsync-path="mkdir -p \"${DOCS_BASE}.oq-engine-new\" && rsync" dot_ddown_adv.json "[email protected]:${DOCS_BASE}.oq-engine-new/.ddown_adv.json"
+          # fi
+
+          # sudo apt update; sudo apt-get install -y texlive-fonts-recommended texlive-latex-extra latexmk gpg
+
+          echo "Make OpenQuake Manual (HTML and PDF)"
+
+          # PAY ATTENTION: engine folder is 'doc', not 'docs'
+          # cd docs/manual && make html && make latexpdf
+          cd docs && make html 2>&1 | tee full.log
+          grep WARNING < full.log > warnings.log
+          echo
+          echo "=== WARNINGS ==="
+          echo
+          cat warnings.log
+          echo
+          echo
+          echo "=== WARNINGS SUMMARY ==="
+          cat warnings.log |  sed 's/.*WARNING/WARNING/g' | sed "s/WARNING: 'myst' cross-reference target not found.*/WARNING: 'myst' cross-reference target not found/g" | sed 's/WARNING: undefined label.*/WARNING: undefined label/g' | sed 's/WARNING: duplicate label.*/WARNING: duplicate label/g' | sed 's/WARNING: Duplicate explicit target name.*/WARNING: Duplicate explicit target name/g' | sort | uniq -c
+
+
+
+          # echo "Uploading OpenQuake Manual (PDF)"
+          # rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --rsync-path="mkdir -p \"${DOCS_BASE}${TARGET}PDF\" && rsync" build/latex/OpenQuakeEngineManual.pdf "[email protected]:${DOCS_BASE}${TARGET}PDF/OpenQuake Manual ${PDF_VER}.pdf"
+          # ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}${PDFDOCS}\"; ln -sf \"../${TARGET}PDF/OpenQuake Manual ${PDF_VER}.pdf\" .'"
+          # if [ "$PDF_VER_LINK" ]; then
+          #     echo "Found PDF_VER_LINK set [$PDF_VER_LINK]"
+          #     ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}manuals/\"; ln -sf \"OpenQuake Manual ${PDF_VER}.pdf\" \"OpenQuake Manual (${PDF_VER_LINK}).pdf\"'"
+          # fi
+
+
+          echo "Uploading New OpenQuake Documentation (HTML)"
+          rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --delete _build/html/ "[email protected]:/var/www/docs.openquake.org/${DOCS_BASE}${TARGET}manual/"
+          echo "------"
+
+
+          # echo "Make Reference Manual (HTML)"
+          # cd ../sphinx && make html
+
+
+          # echo "Upload Reference Manual (HTML)"
+          # rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --delete build/html/ "[email protected]:${DOCS_BASE}${TARGET}reference/"
+          # echo "------"
+
+
+          # echo "Make Advanced Manual (PDF)"
+          # cd ../adv-manual
+          # sed -i "s@#PDF_VER#@${PDF_VER}@g;s@#TARGET#@${DOCS_BASE}${TARGET}@g" index.rst
+          # make clean && make latexpdf
+
+
+          # echo "Upload OpenQuakeforAdvancedUsers (PDF)"
+          # rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --delete build/latex/OpenQuakeforAdvancedUsers.pdf "[email protected]:${DOCS_BASE}${TARGET}PDF/"
+
+          # ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}${PDFDOCS}\"; ln -sf \"../${TARGET}PDF/OpenQuakeforAdvancedUsers.pdf\" \"OpenQuake for Advanced Users ${PDF_VER}.pdf\"'"
+
+          # echo "Make Advanced Manual (HTML)"
+          # cd ../adv-manual && make clean && make html
+
+
+          # echo "Upload OpenQuakeforAdvancedUsers (HTML)"
+          # rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --delete build/html/ "[email protected]:${DOCS_BASE}${TARGET}advanced/"
+
+          # if [ "$BUILD" == "LTS" ]; then
+          #      ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}.oq-engine-new/\"; ln -snf \"$ENG_VER\" LTS'"
+          # fi
+          # if [ "$BUILD" == "release" -o "$BUILD" == "LTS" ]; then
+          #     if [ "$LATEST_VER" == "$ENG_VER" ]; then
+          #         ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}.oq-engine-new/\"; ln -snf \"$ENG_VER\" latest'"
+          #     fi
+          # fi

docs/conf.py

@@ -26,6 +26,7 @@
 html_static_path = ["_static"]
 html_favicon = "_static/OQ-Logo circle_shade.png"
 html_theme_options = {
+    "navigation_with_keys": True,
     "show_nav_level": 2,
     "content_footer_items": ["last-updated"],
     "header_links_before_dropdown": 6,

docs/requirements.txt

@@ -0,0 +1,8 @@
+sphinx==6.2
+pydata-sphinx-theme==0.14.2
+myst-parser==2.0.0
+pygments-csv-lexer==0.1.3
+sphinx-copybutton==0.5.2
+sphinx_design==0.5.0
+sphinxcontrib-images==0.9.4
+sphinxcontrib-youtube==1.4.1