Merge remote-tracking branch 'galaxyproject/master' into transfer-retries-fix

Author: neoformit

.github/workflows/galaxy_framework.yaml

@@ -51,7 +51,7 @@ jobs:
         env:
           GALAXY_TEST_JOB_CONFIG_FILE: ../pulsar/test_data/test_job_conf.yaml
           GALAXY_CONFIG_OVERRIDE_METADATA_STRATEGY: ${{ matrix.metadata-strategy }}
-        continue-on-error: true
+        continue-on-error: ${{ matrix.galaxy-branch == 'master' }}
       - uses: actions/upload-artifact@v2
         with:
           name: Framework test results (${{ matrix.python-version }})

.github/workflows/pulsar.yaml

@@ -24,10 +24,8 @@ jobs:
         include:
         - tox-env: py36-mypy
           python: 3.6
-        - tox-env: py37-mypy
-          python: 3.7
-        - tox-env: py38-mypy
-          python: 3.8
+        - tox-env: py311-mypy
+          python: 3.11
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v3
@@ -36,7 +34,7 @@ jobs:
     - name: Install tox
       run: pip install tox
     - name: Setup pycurl
-      run: sudo apt update; sudo apt install -y libxml2-dev libxslt1-dev libcurl4-openssl-dev python-pycurl openssh-server
+      run: sudo apt update; sudo apt install -y libxml2-dev libxslt1-dev libcurl4-openssl-dev openssh-server
     - name: Run tox
       run: tox -e ${{ matrix.tox-env }}
   test:
@@ -49,21 +47,13 @@ jobs:
           python: 3.6
         - tox-env: py36-test-unit
           python: 3.6
-        - tox-env: py37-test-ci
-          python: 3.7
-        - tox-env: py37-test-unit
-          python: 3.7
-        - tox-env: py38-test-ci
-          python: 3.8
-        - tox-env: py38-test-unit
-          python: 3.8
-        - tox-env: py39-test-ci
-          python: 3.9
-        - tox-env: py39-test-unit
-          python: 3.9
-        - tox-env: py39-install-wheel-no-conda
+        - tox-env: py311-test-ci
+          python: 3.11
+        - tox-env: py311-test-unit
+          python: 3.11
+        - tox-env: py39-install_wheel-no_conda
           python: 3.9
-        - tox-env: py37-install-wheel
+        - tox-env: py37-install_wheel
           python: 3.7
     services:
       job-files:
@@ -94,7 +84,7 @@ jobs:
   ## connect to the host running the Pulsar server for file transfers in stage out/in.
   # tes-test:
   #   name: Run Tests
-  # runs-on: ubuntu-20.04
+  #   runs-on: ubuntu-20.04
   #   strategy:
   #     matrix:
   #       include:

.gitignore

@@ -32,3 +32,4 @@ dependencies
 dependency_resolvers_conf.xml
 job_metrics_conf.xml
 .DS_Store
+.idea

HISTORY.rst

@@ -6,19 +6,34 @@ History
 .. to_doc
 
 ---------------------
-0.15.0.dev1
+0.15.2 (2023-05-02)
 ---------------------
+* Fix Pulsar and Pulsar client reconnection to AMQP server. `Pull Request 324`_
+* Reduce verbosity of timeout exception catching. `Pull Request 325`_
 
-* Updated Galaxy+Pulsar container.
+---------------------
+0.15.1 (2023-04-13)
+---------------------
+* No changes, working around pypi isssue.
+
+---------------------
+0.15.0 (2023-04-13)
+---------------------
+
+* Updated Galaxy+Pulsar container. `Pull Request 306`_
 * Rework container execution - generalize Kubernetes execution to allow it to work without a
-  message queue and to allow TES execution based on pydantic-tes (https://github.com/jmchilton/pydantic-tes).
-* Add documentation and diagrams for container execution scenarios.
+  message queue and to allow TES execution based on pydantic-tes (https://github.com/jmchilton/pydantic-tes). `Pull Request 302`_
+* Add documentation and diagrams for container execution scenarios. `Pull Request 302`_
 * Rework integration tests to use pytest more aggressively.
 * Fixes to CI to run more tests that weren't being executed because Tox was not sending
   environment variables through to pytest.
 * Add option ``amqp_key_prefix`` to direct task queue naming while retaining simple
-  default manager names and such in container scheduling deployments.
-* Various typing and CI fixes.
+  default manager names and such in container scheduling deployments. `Pull Request 315`_
+* Various typing and CI fixes. `Pull Request 312`_, `Pull Request 319`_
+* Fixes for extra_file handling. `Pull Request 318`_
+* Separate tool_stdio and job_stdio handling. `Pull Request 318`_
+* Re-import MEMORY_STATEMENT.sh from Galaxy. `Pull Request 297`_
+* Add support for logging to sentry. `Pull Request 322`_
 
 ---------------------
 0.14.16 (2022-10-04)
@@ -451,6 +466,17 @@ History
 
 
 .. github_links
+
+.. _Pull Request 325: https://github.com/galaxyproject/pulsar/pull/325
+.. _Pull Request 324: https://github.com/galaxyproject/pulsar/pull/324
+.. _Pull Request 322: https://github.com/galaxyproject/pulsar/pull/322
+.. _Pull Request 318: https://github.com/galaxyproject/pulsar/pull/318
+.. _Pull Request 319: https://github.com/galaxyproject/pulsar/pull/319
+.. _Pull Request 312: https://github.com/galaxyproject/pulsar/pull/312
+.. _Pull Request 315: https://github.com/galaxyproject/pulsar/pull/315
+.. _Pull Request 306: https://github.com/galaxyproject/pulsar/pull/306
+.. _Pull Request 297: https://github.com/galaxyproject/pulsar/pull/297
+.. _Pull Request 302: https://github.com/galaxyproject/pulsar/pull/302
 .. _Pull Request 303: https://github.com/galaxyproject/pulsar/pull/303
 .. _Pull Request 301: https://github.com/galaxyproject/pulsar/pull/301
 .. _Pull Request 299: https://github.com/galaxyproject/pulsar/pull/299

app.yml.sample

@@ -89,6 +89,10 @@
 ## kombu.Connection's drain_events() method.
 #amqp_consumer_timeout: 0.2
 
+## publishing messages to the queue may hang if the connection becomes invalid.
+## this value is used as the timeout argument to the producer.publish function.
+#amqp_publish_timeout: 2.0
+
 # AMQP does not guarantee that a published message is received by the AMQP
 # server, so Pulsar can request that the consumer acknowledge messages and will
 # resend them if acknowledgement is not received after a configurable timeout
@@ -120,9 +124,36 @@
 ## Maximum number of seconds to sleep between each retry.
 #amqp_publish_retry_interval_max: 60
 
+
+## configure user authentication/authorization plugins
+## parameters depend on auth type. Authentication plugin should return a username
+## and authorization plugin should authorize this user
+#user_auth:
+#  authentication:
+#    - type: oidc
+#      oidc_jwks_url: https://login.microsoftonline.com/xxx/discovery/v2.0/keys
+#      oidc_provider: azure
+#      oidc_username_in_token: preferred_username
+#      oidc_username_template: *.
+#  authorization:
+#    - type: userlist
+#      userlist_allowed_users:
+#        - xxx
+
 ## *Experimental*. Enable file caching by specifing a directory here. 
 ## Directory used to store incoming file cache. It works fine for HTTP
 ## transfer, have not tested with staging by coping. Also there is no
 ## mechanism for expiring cache so it will grow unbounded without
 ## external clean up.
 #file_cache_dir: cache
+
+## Log to Sentry Sentry is an open source logging and error aggregation
+## platform.  Setting sentry_dsn will enable the Sentry middleware and
+## errors will be sent to the indicated sentry instance.  This
+## connection string is available in your sentry instance under
+## <project_name> -> Settings -> API Keys.
+##sentry_dsn: null
+
+## Determines the minimum log level that will be sent as an event to
+## Sentry. Possible values are DEBUG, INFO, WARNING, ERROR or CRITICAL.
+##sentry_event_level: WARNING

dev-requirements.txt

@@ -18,13 +18,14 @@ sphinx==1.2
 pyflakes
 flake8
 
-mypy
+mypy<=1.0.1  # https://github.com/pydantic/pydantic/issues/5192
 types-paramiko
 types-pkg-resources
 types-PyYAML
 types-pycurl
 types-requests
 types-psutil
+sentry-sdk
 
 # For release
 wheel

docs/configure.rst

@@ -112,6 +112,34 @@ You can consult the `Kombu documentation
 <http://kombu.readthedocs.org/en/latest/reference/kombu.connection.html>`__ for
 even more information.
 
+User Authentication/Authorization
+`````````````
+
+You can configure Pulsar to authenticate user during request processing and check
+if this user is allowed to run a job.
+
+Various authentication/authorization plugins can be configured in `app.yml` to
+do that and plugin parameters depend on auth type. For example, the following
+configuration uses `oidc` plugin for authentication and `userlist` for
+authorization::
+
+    user_auth:
+      authentication:
+        - type: oidc
+          oidc_jwks_url: https://login.microsoftonline.com/xxx/discovery/v2.0/keys
+          oidc_provider: azure
+          oidc_username_in_token: preferred_username
+          oidc_username_template: *.
+      authorization:
+        - type: userlist
+          userlist_allowed_users:
+            - xxx
+
+
+see `plugins folder
+<https://github.com/galaxyproject/pulsar/blob/master/pulsar/user_auth/methods>`_
+for available plugins and their parameters.
+
 Customizing the Pulsar Environment (\*nix only)
 -----------------------------------------------
 

docs/pulsar.client.rst

@@ -77,14 +77,6 @@ pulsar.client.exceptions module
     :undoc-members:
     :show-inheritance:
 
-pulsar.client.interface module
-------------------------------
-
-.. automodule:: pulsar.client.interface
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 pulsar.client.job\_directory module
 -----------------------------------
 
@@ -117,6 +109,14 @@ pulsar.client.path\_mapper module
     :undoc-members:
     :show-inheritance:
 
+pulsar.client.server\_interface module
+--------------------------------------
+
+.. automodule:: pulsar.client.server_interface
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pulsar.client.setup\_handler module
 -----------------------------------
 

docs/pulsar.managers.util.cli.job.rst

@@ -4,6 +4,22 @@ pulsar.managers.util.cli.job package
 Submodules
 ----------
 
+pulsar.managers.util.cli.job.lsf module
+---------------------------------------
+
+.. automodule:: pulsar.managers.util.cli.job.lsf
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+pulsar.managers.util.cli.job.pbs module
+---------------------------------------
+
+.. automodule:: pulsar.managers.util.cli.job.pbs
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pulsar.managers.util.cli.job.slurm module
 -----------------------------------------
 

docs/pulsar.managers.util.rst

@@ -14,6 +14,14 @@ Subpackages
 Submodules
 ----------
 
+pulsar.managers.util.aws\_batch module
+--------------------------------------
+
+.. automodule:: pulsar.managers.util.aws_batch
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pulsar.managers.util.env module
 -------------------------------
 
@@ -38,6 +46,22 @@ pulsar.managers.util.kill module
     :undoc-members:
     :show-inheritance:
 
+pulsar.managers.util.process\_groups module
+-------------------------------------------
+
+.. automodule:: pulsar.managers.util.process_groups
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+pulsar.managers.util.pykube\_util module
+----------------------------------------
+
+.. automodule:: pulsar.managers.util.pykube_util
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pulsar.managers.util.retry module
 ---------------------------------
 
@@ -54,6 +78,14 @@ pulsar.managers.util.sudo module
     :undoc-members:
     :show-inheritance:
 
+pulsar.managers.util.tes module
+-------------------------------
+
+.. automodule:: pulsar.managers.util.tes
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 
 Module contents
 ---------------