Feature/issue 334 drs bulk requests (#365)

briandoconnor · Brian O'Connor · web-flow · commit 775868df9731 · 2023-09-25T13:58:12.000-07:00
* first attempt to add bulk operations

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* bulk operations prototyping

* responding to comments on PR#365

* Create summary.yaml

* Create unresolved.yaml

* Update 200OkDrsObjects.yaml

* working on bulk schema changes

* updates

* updates

* fixed 200 return object for objects endpoint

* fixed 200 return object for objects endpoint

* trying to include passports as a parameter in the bulk body request

* updating the build badge, trying to get passport to show up on bulk

* working to get passports

* trying to get passport field to show up

* trying to figure out adding passports

* trying to get this to build the docs

* adding passports

* working on bulk method for access URLs

* added 413 error

* working on adding bulk options endpoint

* adding error 413 for the bulk options request

* updated logo and version string

* removing passport from bulk opts endpoint since it does not make sense to require authorization

* adding new yaml

* updating travis URL

* updated the security to PassportAuth

* adding a maxBulkRequestLength variable to service info

* adding better description of variables

* adding better description of variables

---------

Co-authored-by: Brian O'Connor &lt;boconnor@nimbusinformatics.com&gt;
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -83,7 +83,7 @@ this will not automatically be built from our Travis.  However, if you
 are a developer and have created a feature branch following the naming
 convention above, you should see automated builds.
 
-Check https://travis-ci.com/ga4gh/data-repository-service-schemas/builds to see the status of the builds.
+Check https://app.travis-ci.com/github/ga4gh/data-repository-service-schemas to see the status of the builds.
 
 Pull Request Voting Process
 ===========================
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
-<img src="https://www.ga4gh.org/wp-content/themes/ga4gh-theme/gfx/GA-logo-horizontal-tag-RGB.svg" alt="GA4GH Logo" style="width: 400px;"/>
+<img src="https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg" alt="GA4GH Logo" style="width: 400px;"/>
 
 # Data Repository Service (DRS) API
 
-<sup>`develop` branch status: </sup>[![Build Status](https://travis-ci.org/ga4gh/data-repository-service-schemas.svg?branch=develop)](https://travis-ci.org/ga4gh/data-repository-service-schemas?branch=develop)
+<sup>`develop` branch status: </sup>[![Build Status](https://app.travis-ci.com/ga4gh/data-repository-service-schemas.svg?branch=develop)](https://app.travis-ci.com/ga4gh/data-repository-service-schemas.svg?branch=develop)
 <a href="https://ga4gh.github.io/data-repository-service-schemas/preview/develop/swagger.yaml"><img src="http://online.swagger.io/validator?url=https://ga4gh.github.io/data-repository-service-schemas/preview/develop/swagger.yaml" alt="Swagger Validator" height="20em" width="72em"></A> [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1405753.svg)](https://doi.org/10.5281/zenodo.1405753)
 
 The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) (GA4GH) is an international coalition, formed to enable the sharing of genomic and clinical data.
diff --git a/openapi/components/parameters/BulkObjectAccessId.yaml b/openapi/components/parameters/BulkObjectAccessId.yaml
@@ -0,0 +1,20 @@
+type: object
+description: The object that contains object_id/access_id tuples
+properties:
+  passports:
+    type: array
+    items:
+      type: string
+  bulk_object_access_ids:
+    type: array
+    items:
+      type: object
+      properties:
+        bulk_object_id:
+          type: string
+          description: DRS object ID
+        bulk_access_ids:
+          type: array
+          description: DRS object access ID
+          items:
+            type: string
diff --git a/openapi/components/parameters/BulkObjectId.yaml b/openapi/components/parameters/BulkObjectId.yaml
@@ -0,0 +1,14 @@
+type: object
+description: The object that contains the DRS object IDs array
+properties:
+  passports:
+    type: array
+    items:
+      type: string
+      example: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJnYTRnaF9wYXNzcG9ydF92MSI6W119.JJ5rN0ktP0qwyZmIPpxmF_p7JsxAZH6L6brUxtad3CM
+    description: the encoded JWT GA4GH Passport that contains embedded Visas.  The overall JWT is signed as are the individual Passport Visas.
+  bulk_object_ids:
+    type: array
+    items:
+      type: string
+    description: An array of ObjectIDs.
diff --git a/openapi/components/parameters/BulkObjectIdNoPassport.yaml b/openapi/components/parameters/BulkObjectIdNoPassport.yaml
@@ -0,0 +1,9 @@
+type: object
+description: The object that contains the DRS object IDs array
+properties:
+  bulk_object_ids:
+    type: array
+    description: DRS object IDs
+    items:
+      type: string
+    description: An array of ObjectIDs.
diff --git a/openapi/components/parameters/BulkObjectSingleAccessId.yaml b/openapi/components/parameters/BulkObjectSingleAccessId.yaml
@@ -0,0 +1,7 @@
+type: object
+description: The object that contains an object_id/access_id tuple
+properties:
+  bulk_object_id:
+    type: string
+  bulk_object_access_id:
+    type: string
diff --git a/openapi/components/responses/200OkAccesses.yaml b/openapi/components/responses/200OkAccesses.yaml
@@ -0,0 +1,14 @@
+description: The `AccessURL` was found successfully
+content:
+  application/json:
+    schema:
+      type: object
+      properties:
+        summary:
+          $ref: '../schemas/summary.yaml'
+        unresolved_drs_objects:
+          $ref: '../schemas/unresolved.yaml'
+        resolved_drs_object_access_urls:
+          type: array
+          items:
+            $ref: '../schemas/BulkAccessURL.yaml'
diff --git a/openapi/components/responses/200OkBulkAuthorizations.yaml b/openapi/components/responses/200OkBulkAuthorizations.yaml
@@ -0,0 +1,14 @@
+description: "`Authorizations` were found successfully"
+content:
+  application/json:
+    schema:
+      type: object
+      properties:
+        summary:
+          $ref: '../schemas/summary.yaml'
+        unresolved_drs_objects:
+          $ref: '../schemas/unresolved.yaml'
+        resolved_drs_object:
+          type: array
+          items:
+            $ref: '../schemas/Authorizations.yaml'
diff --git a/openapi/components/responses/200OkDrsObjects.yaml b/openapi/components/responses/200OkDrsObjects.yaml
@@ -0,0 +1,14 @@
+description: The `DrsObjects` were found successfully
+content:
+  application/json:
+    schema:
+      type: object
+      properties:
+        summary:
+          $ref: '../schemas/summary.yaml'
+        unresolved_drs_objects:
+          $ref: '../schemas/unresolved.yaml'
+        resolved_drs_object:
+          type: array
+          items:
+            $ref: '../schemas/DrsObject.yaml'
diff --git a/openapi/components/responses/404NotFoundDrsObject.yaml b/openapi/components/responses/404NotFoundDrsObject.yaml
@@ -2,4 +2,4 @@ description: The requested `DrsObject` wasn't found.
 content:
   application/json:
     schema:
-      $ref: '../schemas/Error.yaml'
+      $ref: '../schemas/Error.yaml'
diff --git a/openapi/components/responses/413RequestTooLarge.yaml b/openapi/components/responses/413RequestTooLarge.yaml
@@ -0,0 +1,5 @@
+description: The bulk request is too large.
+content:
+  application/json:
+    schema:
+      $ref: '../schemas/Error.yaml'
diff --git a/openapi/components/schemas/Authorizations.yaml b/openapi/components/schemas/Authorizations.yaml
@@ -1,5 +1,7 @@
 type: object
 properties:
+  drs_object_id:
+    type: string
   supported_types:
     type: array
     items:
diff --git a/openapi/components/schemas/BulkAccessURL.yaml b/openapi/components/schemas/BulkAccessURL.yaml
@@ -0,0 +1,19 @@
+type: object
+required:
+  - url
+properties:
+  drs_object_id:
+    type: string
+  drs_access_id:
+    type: string
+  url:
+    type: string
+    description: A fully resolvable URL that can be used to fetch the actual object bytes.
+  headers:
+    type: array
+    items:
+      type: string
+    description: >-
+      An optional list of headers to include in the HTTP request to `url`.
+      These headers can be used to provide auth tokens required to fetch the object bytes.
+    example: 'Authorization: Basic Z2E0Z2g6ZHJz'
diff --git a/openapi/components/schemas/DrsService.yaml b/openapi/components/schemas/DrsService.yaml
@@ -2,6 +2,10 @@ type: object
 required:
   - type
 properties:
+  maxBulkRequestLength:
+    type: integer
+    required: true
+    description: The max length the bullk request endpoints can handle (>= 1) before generating a 413 error e.g. how long can the arrays bulk_object_ids and bulk_object_access_ids be for this server.  
   type:
     type: object
     required:
diff --git a/openapi/components/schemas/summary.yaml b/openapi/components/schemas/summary.yaml
@@ -0,0 +1,12 @@
+type: object
+description: A summary of what was resolved.
+properties:
+  requested:
+    type: integer
+    description: Number of items requested.
+  resolved:
+    type: integer
+    description: Number of objects resolved.
+  unresolved:
+    type: integer
+    description: Number of objects not resolved.
diff --git a/openapi/components/schemas/unresolved.yaml b/openapi/components/schemas/unresolved.yaml
@@ -0,0 +1,11 @@
+type: array
+description: Error codes for each unresolved drs objects.
+items:
+  type: object
+  properties:
+    error_code:
+      type: integer
+    object_ids:
+      type: array
+      items:
+        type: string
diff --git a/openapi/data_repository_service.openapi.yaml b/openapi/data_repository_service.openapi.yaml
@@ -1,9 +1,9 @@
 openapi: 3.0.3
 info:
   title: Data Repository Service
-  version: 1.3.0
+  version: 1.4.0
   x-logo:
-    url: 'https://www.ga4gh.org/wp-content/themes/ga4gh-theme/gfx/GA-logo-horizontal-tag-RGB.svg'
+    url: 'https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg'
   termsOfService: 'https://www.ga4gh.org/terms-and-conditions/'
   contact:
     name: GA4GH Cloud Work Stream
@@ -115,8 +115,12 @@ paths:
     $ref: ./paths/service-info.yaml
   /objects/{object_id}:
     $ref: ./paths/objects@{object_id}.yaml
+  /objects:
+    $ref: ./paths/bulkobjects@{object_id}.yaml
   /objects/{object_id}/access/{access_id}:
     $ref: ./paths/objects@{object_id}@access@{access_id}.yaml
+  /objects/access:
+    $ref: ./paths/bulkobjects@access@{access_id}.yaml
 components:
   securitySchemes:
     BasicAuth:
diff --git a/openapi/paths/bulkobjects@access@{access_id}.yaml b/openapi/paths/bulkobjects@access@{access_id}.yaml
@@ -0,0 +1,38 @@
+post:
+  summary: Get URLs for fetching bytes from multiple objects with an optional Passport(s).
+  description: >-
+    Returns an array of URL objects that can be used to fetch the bytes of multiple `DrsObject`s.
+
+    This method only needs to be called when using an `AccessMethod` that contains an `access_id`
+    (e.g., for servers that use signed URLs for fetching object bytes).
+
+    Currently this is limited to use passports (one or more) or a single bearer token, so make sure your bulk request is for objects that all use the same passports/token.
+  operationId: GetBulkAccessURL
+  security:
+    - PassportAuth: []
+  responses:
+    200:
+      $ref: '../components/responses/200OkAccesses.yaml'
+    202:
+      $ref: '../components/responses/202Accepted.yaml'
+    400:
+      $ref: '../components/responses/400BadRequest.yaml'
+    401:
+      $ref: '../components/responses/401Unauthorized.yaml'
+    403:
+      $ref: '../components/responses/403Forbidden.yaml'
+    404:
+      $ref: '../components/responses/404NotFoundAccess.yaml'
+    413:
+      $ref: '../components/responses/413RequestTooLarge.yaml'
+    500:
+      $ref: '../components/responses/500InternalServerError.yaml'
+  tags:
+    - Objects
+  x-swagger-router-controller: ga4gh.drs.server
+  requestBody:
+    required: true
+    content:
+      application/json:
+        schema:
+          $ref: '../components/parameters/BulkObjectAccessId.yaml'
diff --git a/openapi/paths/bulkobjects@{object_id}.yaml b/openapi/paths/bulkobjects@{object_id}.yaml
@@ -0,0 +1,67 @@
+options:
+  summary: Get Authorization info about multiple DrsObjects.
+  security:
+    - {}
+  description: >-
+    Returns a structure that contains for each DrsObjects a list of `Authorizations` that can be used to determine how to authorize requests to `GetObject` or `PostObject` (or bulk equivalents).
+  operationId: OptionsBulkObject
+  responses:
+    200:
+      $ref: '../components/responses/200OkBulkAuthorizations.yaml'
+    204:
+      $ref: '../components/responses/AuthorizationsNotSupported.yaml'
+    400:
+      $ref: '../components/responses/400BadRequest.yaml'
+    404:
+      $ref: '../components/responses/404NotFoundDrsObject.yaml'
+    405:
+      $ref: '../components/responses/AuthorizationsNotSupported.yaml'
+    413:
+      $ref: '../components/responses/413RequestTooLarge.yaml'
+    500:
+      $ref: '../components/responses/500InternalServerError.yaml'
+  tags:
+    - Objects
+  x-swagger-router-controller: ga4gh.drs.server
+  requestBody:
+    required: true
+    content:
+      application/json:
+        schema:
+          $ref: '../components/parameters/BulkObjectIdNoPassport.yaml'
+
+post:
+  summary: Get info about multiple DrsObjects with an optional Passport(s).
+  description: >-
+    Returns an array of object metadata, and a list of access methods that can be used to fetch objects' bytes.  Currently this is limited to use passports (one or more) or a single bearer token, so make sure your bulk request is for objects that all use the same passports/token.
+  operationId: GetBulkObjects
+  security:
+    - PassportAuth: []
+  parameters:
+    - $ref: '../components/parameters/Expand.yaml'
+  responses:
+    200:
+      $ref: '../components/responses/200OkDrsObjects.yaml'
+    202:
+      $ref: '../components/responses/202Accepted.yaml'
+    400:
+      $ref: '../components/responses/400BadRequest.yaml'
+    401:
+      $ref: '../components/responses/401Unauthorized.yaml'
+    403:
+      $ref: '../components/responses/403Forbidden.yaml'
+    404:
+      $ref: '../components/responses/404NotFoundDrsObject.yaml'
+    413:
+      $ref: '../components/responses/413RequestTooLarge.yaml'
+    500:
+      $ref: '../components/responses/500InternalServerError.yaml'
+  tags:
+    - Objects
+  x-swagger-router-controller: ga4gh.drs.server
+  requestBody:
+    required: true
+    content:
+      application/json:
+        schema:
+          $ref: '../components/parameters/BulkObjectId.yaml'
diff --git a/openapi/tags/ServiceRegistry.md b/openapi/tags/ServiceRegistry.md
@@ -17,7 +17,7 @@ Example listing of a DRS API registration from a service registry's `/services`
         "type": {
             "group": "org.ga4gh",
             "artifact": "drs",
-            "version": "1.3.0"
+            "version": "1.4.0"
         },
         "description": "The Data Repository Service (DRS) API ...",
         "organization": {
diff --git a/pages/more-background-on-compact-identifiers/openapi.yaml b/pages/more-background-on-compact-identifiers/openapi.yaml
@@ -1,9 +1,9 @@
 openapi: 3.0.3
 info:
   title: More Background on Compact Identifiers
-  version: 1.3.0
+  version: 1.4.0
   x-logo:
-    url: 'https://www.ga4gh.org/wp-content/themes/ga4gh-theme/gfx/GA-logo-horizontal-tag-RGB.svg'
+    url: 'https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg'
   termsOfService: 'https://www.ga4gh.org/terms-and-conditions/'
   contact:
     name: GA4GH Cloud Work Stream
