diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 6f83e1d8f6..3bec445c1c 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -228,7 +228,7 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '7.2' + 'version': '7.3' }, 'tags': [ { @@ -1273,14 +1273,15 @@ def get_summary(): authentication=request.authentication) -def manifest_route(*, fetch: bool, initiate: bool): +def manifest_route(*, fetch: bool, method: str): + initiate = method in {'PUT', 'POST'} return app.route( # The path parameter could be a token *or* an object key, but we don't # want to complicate the API with this detail ('/fetch' if fetch else '') + ('/manifest/files' if initiate else '/manifest/files/{token}'), # The initial PUT request is idempotent. - methods=['PUT' if initiate else 'GET'], + methods=[method], content_types=['application/json', 'application/x-www-form-urlencoded'], interactive=fetch, cors=True, @@ -1301,7 +1302,7 @@ def manifest_route(*, fetch: bool, initiate: bool): ) + ( ' via XHR' if fetch else '' ), - 'description': fd(''' + 'description': fd(f''' Create a manifest preparation job, returning either - a 301 redirect to the URL of the status of that job or @@ -1309,9 +1310,9 @@ def manifest_route(*, fetch: bool, initiate: bool): - a 302 redirect to the URL of an already prepared manifest. This endpoint is not suitable for interactive use via the - Swagger UI. Please use [PUT /fetch/manifest/files][1] instead. + Swagger UI. Please use [{method} /fetch/manifest/files][1] instead. - [1]: #operations-Manifests-put_fetch_manifest_files + [1]: #operations-Manifests-{method.lower()}_fetch_manifest_files ''') + parameter_hoisting_note if initiate and not fetch else fd(''' Check on the status of an ongoing manifest preparation job, returning either @@ -1326,15 +1327,15 @@ def manifest_route(*, fetch: bool, initiate: bool): instead. [1]: #operations-Manifests-get_fetch_manifest_files__token_ - ''') if not initiate and not fetch else fd(''' + ''') if not initiate and not fetch else fd(f''' Create a manifest preparation job, returning a 200 status response whose JSON body emulates the HTTP headers that would be - found in a response to an equivalent request to the [PUT + found in a response to an equivalent request to the [{method} /manifest/files][1] endpoint. Whenever client-side JavaScript code is used in a web application to request the preparation of a manifest from Azul, - this endpoint should be used instead of [PUT + this endpoint should be used instead of [{method} /manifest/files][1]. This way, the client can use XHR to make the request, retaining full control over the handling of redirects and enabling the client to bypass certain limitations @@ -1344,7 +1345,7 @@ def manifest_route(*, fetch: bool, initiate: bool): upper limit on the number of consecutive redirects, before the manifest generation job is done. - [1]: #operations-Manifests-put_manifest_files + [1]: #operations-Manifests-{method.lower()}_manifest_files ''') + parameter_hoisting_note if initiate and fetch else fd(''' Check on the status of an ongoing manifest preparation job, returning a 200 status response whose JSON body emulates the @@ -1485,10 +1486,10 @@ def manifest_route(*, fetch: bool, initiate: bool): For a detailed description of these properties see the documentation for the respective response headers - documented under ''') + (fd(''' - [PUT /manifest/files][1]. + documented under ''') + (fd(f''' + [{method} /manifest/files][1]. - [1]: #operations-Manifests-put_manifest_files + [1]: #operations-Manifests-{method.lower()}_manifest_files ''') if initiate else fd(''' [GET /manifest/files/{token}][1]. @@ -1530,28 +1531,32 @@ def manifest_route(*, fetch: bool, initiate: bool): ) -@manifest_route(fetch=False, initiate=True) +@manifest_route(fetch=False, method='PUT') +@manifest_route(fetch=False, method='POST') def file_manifest(): return _file_manifest(fetch=False) -@manifest_route(fetch=False, initiate=False) +@manifest_route(fetch=False, method='GET') def file_manifest_with_token(token: str): return _file_manifest(fetch=False, token_or_key=token) -@manifest_route(fetch=True, initiate=True) +@manifest_route(fetch=True, method='PUT') +@manifest_route(fetch=True, method='POST') def fetch_file_manifest(): return _file_manifest(fetch=True) -@manifest_route(fetch=True, initiate=False) +@manifest_route(fetch=True, method='GET') def fetch_file_manifest_with_token(token: str): return _file_manifest(fetch=True, token_or_key=token) def _file_manifest(fetch: bool, token_or_key: Optional[str] = None): request = app.current_request + require(request.method != 'POST' or request.raw_body == b'', + 'The body must be empty for a POST request to this endpoint.') query_params = request.query_params or {} _hoist_parameters(query_params, request) if token_or_key is None: diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index aaa374a3f2..80437aaba3 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "azul_service", "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n \"projects\": [\n {\n \"projectTitle\": \"Project One\"\n \"laboratory\": ...,\n ...\n },\n {\n \"projectTitle\": \"Project Two\"\n \"laboratory\": ...,\n ...\n }\n ],\n \"files\": [\n {\n \"format\": \"pdf\",\n \"name\": \"Team description.pdf\",\n ...\n }\n ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n", - "version": "7.2" + "version": "7.3" }, "tags": [ { @@ -8168,12 +8168,12 @@ } }, "/manifest/files": { - "put": { + "post": { "tags": [ "Manifests" ], "summary": "Initiate the preparation of a manifest", - "description": "\nCreate a manifest preparation job, returning either\n\n- a 301 redirect to the URL of the status of that job or\n\n- a 302 redirect to the URL of an already prepared manifest.\n\nThis endpoint is not suitable for interactive use via the\nSwagger UI. Please use [PUT /fetch/manifest/files][1] instead.\n\n[1]: #operations-Manifests-put_fetch_manifest_files\n\nAny of the query parameters documented below can alternatively be passed as\na property of a JSON object in the body of the request. This can be useful\nin case the value of the `filters` query parameter causes the URL to exceed\nthe maximum length of 8192 characters, resulting in a 413 Request Entity Too\nLarge response.\n\nThe request `GET /index/foo?filters={\u2026}`, for example, is equivalent to a\n`POST /index/foo` request with the body `{\"filters\": {\u2026}}`.\n", + "description": "\nCreate a manifest preparation job, returning either\n\n- a 301 redirect to the URL of the status of that job or\n\n- a 302 redirect to the URL of an already prepared manifest.\n\nThis endpoint is not suitable for interactive use via the\nSwagger UI. Please use [POST /fetch/manifest/files][1] instead.\n\n[1]: #operations-Manifests-post_fetch_manifest_files\n\nAny of the query parameters documented below can alternatively be passed as\na property of a JSON object in the body of the request. This can be useful\nin case the value of the `filters` query parameter causes the URL to exceed\nthe maximum length of 8192 characters, resulting in a 413 Request Entity Too\nLarge response.\n\nThe request `GET /index/foo?filters={\u2026}`, for example, is equivalent to a\n`POST /index/foo` request with the body `{\"filters\": {\u2026}}`.\n", "parameters": [ { "name": "catalog", @@ -9490,7 +9490,7 @@ "description": "\nA redirect indicating that the manifest preparation job\nhas started. Wait for\nthe recommended number of seconds (see `Retry-After`\nheader) and then follow the redirect to check the status\nof that job.\n", "headers": { "Location": { - "description": "\nThe URL of the manifest preparation job at\nthe [`GET\n /manifest/files/{token}`][2] endpoint.\n\n [2]: #operations-Manifests-get_fetch_manifest_files_token\n", + "description": "\nThe URL of the manifest preparation job at\n\nthe `GET /manifest/files/{token}` endpoint.\n", "schema": { "type": "string", "format": "url" @@ -9517,65 +9517,2780 @@ } } } - } - }, - "/manifest/files/{token}": { - "parameters": [ - { - "name": "token", - "in": "path", - "required": true, - "schema": { - "type": "string" - }, - "description": "\nAn opaque string representing the manifest preparation job\n" - } - ], - "get": { + }, + "put": { "tags": [ "Manifests" ], - "summary": "Determine status of a manifest preparation job", - "description": "\nCheck on the status of an ongoing manifest preparation job,\nreturning either\n\n- a 301 redirect to this endpoint if the manifest job is still\n running\n\n- a 302 redirect to the URL of the completed manifest.\n\nThis endpoint is not suitable for interactive use via the\nSwagger UI. Please use [GET /fetch/manifest/files/{token}][1]\ninstead.\n\n[1]: #operations-Manifests-get_fetch_manifest_files\n", - "parameters": [], - "responses": { - "301": { - "description": "\nA redirect indicating that the manifest preparation job\nis running. Wait for\nthe recommended number of seconds (see `Retry-After`\nheader) and then follow the redirect to check the status\nof the job again.\n", - "headers": { - "Location": { - "description": "\nThe URL of this endpoint\n", - "schema": { - "type": "string", - "format": "url" - } - }, - "Retry-After": { - "description": "\nThe recommended number of seconds to wait before\nrequesting the URL specified in the `Location`\nheader\n", - "schema": { - "type": "string" - } - } - } + "summary": "Initiate the preparation of a manifest", + "description": "\nCreate a manifest preparation job, returning either\n\n- a 301 redirect to the URL of the status of that job or\n\n- a 302 redirect to the URL of an already prepared manifest.\n\nThis endpoint is not suitable for interactive use via the\nSwagger UI. Please use [PUT /fetch/manifest/files][1] instead.\n\n[1]: #operations-Manifests-put_fetch_manifest_files\n\nAny of the query parameters documented below can alternatively be passed as\na property of a JSON object in the body of the request. This can be useful\nin case the value of the `filters` query parameter causes the URL to exceed\nthe maximum length of 8192 characters, resulting in a 413 Request Entity Too\nLarge response.\n\nThe request `GET /index/foo?filters={\u2026}`, for example, is equivalent to a\n`POST /index/foo` request with the body `{\"filters\": {\u2026}}`.\n", + "parameters": [ + { + "name": "catalog", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "dcp2" + ], + "default": "dcp2" + }, + "description": "The name of the catalog to query." }, - "302": { - "description": "\nA redirect indicating that the manifest preparation job\nis now done. Immediately\nfollow the redirect to obtain the manifest contents.\n\nThe response body contains, for a number of commonly\nused shells, a command line suitable for downloading the\nmanifest.\n", - "headers": { - "Location": { - "description": "The URL of the manifest.\n Clients should not make any assumptions about\n any parts of the returned domain, except that\n the scheme will be `https`.\n", + { + "name": "filters", + "in": "query", + "required": false, + "content": { + "application/json": { "schema": { - "type": "string", - "format": "url" - } - } - } - }, - "410": { - "description": "\nThe manifest preparation job has expired. Request a\nnew preparation using the `PUT /manifest/files`\nendpoint.\n" - } - } - } - }, - "/fetch/manifest/files": { + "type": "object", + "properties": { + "accessions": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "nullable": true + }, + "accession": { + "type": "string", + "nullable": true + } + }, + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateLastModifiedDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateSubmissionDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateUpdateDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "assayType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "biologicalSex": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bionetworkName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bundleUuid": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bundleVersion": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "cellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "cellLineType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "contactName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "contentDescription": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "developmentStage": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "donorCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "donorDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "effectiveCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "effectiveOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "entryId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileFormat": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileSize": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "fileSource": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileVersion": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "genusSpecies": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "institution": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "instrumentManufacturerModel": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "isIntermediate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "isTissueAtlasProject": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "laboratory": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "lastModifiedDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "libraryConstructionApproach": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "matrixCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "modelOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "modelOrganPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "nucleicAcidSource": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organ": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organismAge": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "required": [ + "value", + "unit" + ], + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organismAgeRange": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "contains": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "number", + "format": "double" + }, + { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + ] + } + } + }, + "required": [ + "contains" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "intersects": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "intersects" + ], + "additionalProperties": false + } + ] + }, + "pairedEnd": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "preservationMethod": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "project": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectDescription": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectEstimatedCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "projectId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectTitle": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "publicationTitle": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleEntityType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "selectedCellType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sourceId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sourceSpec": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenOrganPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "submissionDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "tissueAtlas": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "atlas": { + "type": "string", + "nullable": true + }, + "version": { + "type": "string", + "nullable": true + } + }, + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "updateDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "workflow": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + } + }, + "default": "{}", + "example": { + "cellCount": { + "within": [ + [ + 10000, + 1000000000 + ] + ] + } + } + } + } + }, + "description": "\nCriteria to filter entities from the search results.\n\nEach filter consists of a field name, a relation (relational operator),\nand an array of field values. The available relations are \"is\",\n\"within\", \"contains\", and \"intersects\". Multiple filters are combined\nusing \"and\" logic. An entity must match all filters to be included in\nthe response. How multiple field values within a single filter are\ncombined depends on the relation.\n\nFor the \"is\" relation, multiple values are combined using \"or\" logic.\nFor example, `{\"fileFormat\": {\"is\": [\"fastq\", \"fastq.gz\"]}}` selects\nentities where the file format is either \"fastq\" or \"fastq.gz\". For the\n\"within\", \"intersects\", and \"contains\" relations, the field values must\ncome in nested pairs specifying upper and lower bounds, and multiple\npairs are combined using \"and\" logic. For example, `{\"donorCount\":\n{\"within\": [[1,5], [5,10]]}}` selects entities whose donor organism\ncount falls within both ranges, i.e., is exactly 5.\n\nThe accessions field supports filtering for a specific accession and/or\nnamespace within a project. For example, `{\"accessions\": {\"is\": [\n{\"namespace\":\"array_express\"}]}}` will filter for projects that have an\n`array_express` accession. Similarly, `{\"accessions\": {\"is\": [\n{\"accession\":\"ERP112843\"}]}}` will filter for projects that have the\naccession `ERP112843` while `{\"accessions\": {\"is\": [\n{\"namespace\":\"array_express\", \"accession\": \"E-AAAA-00\"}]}}` will filter\nfor projects that match both values.\n\nThe organismAge field is special in that it contains two property keys:\nvalue and unit. For example, `{\"organismAge\": {\"is\": [{\"value\": \"20\",\n\"unit\": \"year\"}]}}`. Both keys are required. `{\"organismAge\": {\"is\":\n[null]}}` selects entities that have no organism age.\n\nSupported field names are: accessions, aggregateLastModifiedDate, aggregateSubmissionDate, aggregateUpdateDate, assayType, biologicalSex, bionetworkName, bundleUuid, bundleVersion, cellCount, cellLineType, contactName, contentDescription, developmentStage, donorCount, donorDisease, effectiveCellCount, effectiveOrgan, entryId, fileFormat, fileId, fileName, fileSize, fileSource, fileVersion, genusSpecies, institution, instrumentManufacturerModel, isIntermediate, isTissueAtlasProject, laboratory, lastModifiedDate, libraryConstructionApproach, matrixCellCount, modelOrgan, modelOrganPart, nucleicAcidSource, organ, organPart, organismAge, organismAgeRange, pairedEnd, preservationMethod, project, projectDescription, projectEstimatedCellCount, projectId, projectTitle, publicationTitle, sampleDisease, sampleEntityType, sampleId, selectedCellType, sourceId, sourceSpec, specimenDisease, specimenOrgan, specimenOrganPart, submissionDate, tissueAtlas, updateDate, workflow\n" + }, + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "compact", + "terra.bdbag", + "terra.pfb", + "curl", + "verbatim.jsonl" + ] + }, + "description": "\nThe desired format of the output.\n\n- `compact` (the default) for a compact,\n tab-separated manifest\n\n- `terra.bdbag` for a manifest in the\n [BDBag format][1]. This provides a ZIP file containing two\n manifests: one for Participants (aka Donors) and one for\n Samples (aka Specimens). For more on the format of the\n manifests see [documentation here][2].\n\n- `terra.pfb` for a manifest in the [PFB\n format][3]. This format is mainly used for exporting data to\n Terra.\n\n- `curl` for a [curl configuration\n file][4] manifest. This manifest can be used with the curl\n program to download all the files listed in the manifest.\n\n- `verbatim.jsonl` for a verbatim\n manifest in [JSONL][5] format. Each line contains an\n unaltered metadata entity from the underlying repository.\n\n[1]: https://bd2k.ini.usc.edu/tools/bdbag/\n\n[2]: https://software.broadinstitute.org/firecloud/documentation/article?id=10954\n\n[3]: https://github.com/uc-cdis/pypfb\n\n[4]: https://curl.haxx.se/docs/manpage.html#-K\n\n[5]: https://jsonlines.org/\n" + } + ], + "responses": { + "301": { + "description": "\nA redirect indicating that the manifest preparation job\nhas started. Wait for\nthe recommended number of seconds (see `Retry-After`\nheader) and then follow the redirect to check the status\nof that job.\n", + "headers": { + "Location": { + "description": "\nThe URL of the manifest preparation job at\n\nthe `GET /manifest/files/{token}` endpoint.\n", + "schema": { + "type": "string", + "format": "url" + } + }, + "Retry-After": { + "description": "\nThe recommended number of seconds to wait before\nrequesting the URL specified in the `Location`\nheader\n", + "schema": { + "type": "string" + } + } + } + }, + "302": { + "description": "\nA redirect indicating that the manifest preparation job\nis already done. Immediately\nfollow the redirect to obtain the manifest contents.\n\nThe response body contains, for a number of commonly\nused shells, a command line suitable for downloading the\nmanifest.\n", + "headers": { + "Location": { + "description": "The URL of the manifest.\n Clients should not make any assumptions about\n any parts of the returned domain, except that\n the scheme will be `https`.\n", + "schema": { + "type": "string", + "format": "url" + } + } + } + } + } + } + }, + "/manifest/files/{token}": { + "parameters": [ + { + "name": "token", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "\nAn opaque string representing the manifest preparation job\n" + } + ], + "get": { + "tags": [ + "Manifests" + ], + "summary": "Determine status of a manifest preparation job", + "description": "\nCheck on the status of an ongoing manifest preparation job,\nreturning either\n\n- a 301 redirect to this endpoint if the manifest job is still\n running\n\n- a 302 redirect to the URL of the completed manifest.\n\nThis endpoint is not suitable for interactive use via the\nSwagger UI. Please use [GET /fetch/manifest/files/{token}][1]\ninstead.\n\n[1]: #operations-Manifests-get_fetch_manifest_files__token_\n", + "parameters": [], + "responses": { + "301": { + "description": "\nA redirect indicating that the manifest preparation job\nis running. Wait for\nthe recommended number of seconds (see `Retry-After`\nheader) and then follow the redirect to check the status\nof the job again.\n", + "headers": { + "Location": { + "description": "\nThe URL of this endpoint\n", + "schema": { + "type": "string", + "format": "url" + } + }, + "Retry-After": { + "description": "\nThe recommended number of seconds to wait before\nrequesting the URL specified in the `Location`\nheader\n", + "schema": { + "type": "string" + } + } + } + }, + "302": { + "description": "\nA redirect indicating that the manifest preparation job\nis now done. Immediately\nfollow the redirect to obtain the manifest contents.\n\nThe response body contains, for a number of commonly\nused shells, a command line suitable for downloading the\nmanifest.\n", + "headers": { + "Location": { + "description": "The URL of the manifest.\n Clients should not make any assumptions about\n any parts of the returned domain, except that\n the scheme will be `https`.\n", + "schema": { + "type": "string", + "format": "url" + } + } + } + }, + "410": { + "description": "\nThe manifest preparation job has expired. Request a\nnew preparation using the `PUT /manifest/files`\nendpoint.\n" + } + } + } + }, + "/fetch/manifest/files": { + "post": { + "tags": [ + "Manifests" + ], + "summary": "Initiate the preparation of a manifest via XHR", + "description": "\nCreate a manifest preparation job, returning a 200 status\nresponse whose JSON body emulates the HTTP headers that would be\nfound in a response to an equivalent request to the [POST\n/manifest/files][1] endpoint.\n\nWhenever client-side JavaScript code is used in a web\napplication to request the preparation of a manifest from Azul,\nthis endpoint should be used instead of [POST\n/manifest/files][1]. This way, the client can use XHR to make\nthe request, retaining full control over the handling of\nredirects and enabling the client to bypass certain limitations\non the native handling of redirects in web browsers. For\nexample, most browsers ignore the `Retry-After` header in\nredirect responses, causing them to prematurely exhaust the\nupper limit on the number of consecutive redirects, before the\nmanifest generation job is done.\n\n[1]: #operations-Manifests-post_manifest_files\n\nAny of the query parameters documented below can alternatively be passed as\na property of a JSON object in the body of the request. This can be useful\nin case the value of the `filters` query parameter causes the URL to exceed\nthe maximum length of 8192 characters, resulting in a 413 Request Entity Too\nLarge response.\n\nThe request `GET /index/foo?filters={\u2026}`, for example, is equivalent to a\n`POST /index/foo` request with the body `{\"filters\": {\u2026}}`.\n", + "parameters": [ + { + "name": "catalog", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "dcp2" + ], + "default": "dcp2" + }, + "description": "The name of the catalog to query." + }, + { + "name": "filters", + "in": "query", + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "accessions": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "nullable": true + }, + "accession": { + "type": "string", + "nullable": true + } + }, + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateLastModifiedDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateSubmissionDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "aggregateUpdateDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "assayType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "biologicalSex": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bionetworkName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bundleUuid": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "bundleVersion": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "cellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "cellLineType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "contactName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "contentDescription": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "developmentStage": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "donorCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "donorDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "effectiveCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "effectiveOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "entryId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileFormat": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileName": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileSize": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "fileSource": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "fileVersion": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "genusSpecies": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "institution": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "instrumentManufacturerModel": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "isIntermediate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "isTissueAtlasProject": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "laboratory": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "lastModifiedDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "libraryConstructionApproach": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "matrixCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "modelOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "modelOrganPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "nucleicAcidSource": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organ": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organismAge": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "required": [ + "value", + "unit" + ], + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "organismAgeRange": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "contains": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "number", + "format": "double" + }, + { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + ] + } + } + }, + "required": [ + "contains" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "intersects": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "intersects" + ], + "additionalProperties": false + } + ] + }, + "pairedEnd": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "boolean", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "preservationMethod": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "project": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectDescription": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectEstimatedCellCount": { + "oneOf": [ + { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "integer", + "format": "int64", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "within": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "required": [ + "within" + ], + "additionalProperties": false + } + ] + }, + "projectId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "projectTitle": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "publicationTitle": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleEntityType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sampleId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "selectedCellType": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sourceId": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "sourceSpec": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenDisease": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenOrgan": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "specimenOrganPart": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "submissionDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "tissueAtlas": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "object", + "properties": { + "atlas": { + "type": "string", + "nullable": true + }, + "version": { + "type": "string", + "nullable": true + } + }, + "additionalProperties": false + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "updateDate": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + }, + "workflow": { + "type": "object", + "properties": { + "is": { + "type": "array", + "items": { + "type": "string", + "nullable": true + } + } + }, + "required": [ + "is" + ], + "additionalProperties": false + } + }, + "default": "{}", + "example": { + "cellCount": { + "within": [ + [ + 10000, + 1000000000 + ] + ] + } + } + } + } + }, + "description": "\nCriteria to filter entities from the search results.\n\nEach filter consists of a field name, a relation (relational operator),\nand an array of field values. The available relations are \"is\",\n\"within\", \"contains\", and \"intersects\". Multiple filters are combined\nusing \"and\" logic. An entity must match all filters to be included in\nthe response. How multiple field values within a single filter are\ncombined depends on the relation.\n\nFor the \"is\" relation, multiple values are combined using \"or\" logic.\nFor example, `{\"fileFormat\": {\"is\": [\"fastq\", \"fastq.gz\"]}}` selects\nentities where the file format is either \"fastq\" or \"fastq.gz\". For the\n\"within\", \"intersects\", and \"contains\" relations, the field values must\ncome in nested pairs specifying upper and lower bounds, and multiple\npairs are combined using \"and\" logic. For example, `{\"donorCount\":\n{\"within\": [[1,5], [5,10]]}}` selects entities whose donor organism\ncount falls within both ranges, i.e., is exactly 5.\n\nThe accessions field supports filtering for a specific accession and/or\nnamespace within a project. For example, `{\"accessions\": {\"is\": [\n{\"namespace\":\"array_express\"}]}}` will filter for projects that have an\n`array_express` accession. Similarly, `{\"accessions\": {\"is\": [\n{\"accession\":\"ERP112843\"}]}}` will filter for projects that have the\naccession `ERP112843` while `{\"accessions\": {\"is\": [\n{\"namespace\":\"array_express\", \"accession\": \"E-AAAA-00\"}]}}` will filter\nfor projects that match both values.\n\nThe organismAge field is special in that it contains two property keys:\nvalue and unit. For example, `{\"organismAge\": {\"is\": [{\"value\": \"20\",\n\"unit\": \"year\"}]}}`. Both keys are required. `{\"organismAge\": {\"is\":\n[null]}}` selects entities that have no organism age.\n\nSupported field names are: accessions, aggregateLastModifiedDate, aggregateSubmissionDate, aggregateUpdateDate, assayType, biologicalSex, bionetworkName, bundleUuid, bundleVersion, cellCount, cellLineType, contactName, contentDescription, developmentStage, donorCount, donorDisease, effectiveCellCount, effectiveOrgan, entryId, fileFormat, fileId, fileName, fileSize, fileSource, fileVersion, genusSpecies, institution, instrumentManufacturerModel, isIntermediate, isTissueAtlasProject, laboratory, lastModifiedDate, libraryConstructionApproach, matrixCellCount, modelOrgan, modelOrganPart, nucleicAcidSource, organ, organPart, organismAge, organismAgeRange, pairedEnd, preservationMethod, project, projectDescription, projectEstimatedCellCount, projectId, projectTitle, publicationTitle, sampleDisease, sampleEntityType, sampleId, selectedCellType, sourceId, sourceSpec, specimenDisease, specimenOrgan, specimenOrganPart, submissionDate, tissueAtlas, updateDate, workflow\n" + }, + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "compact", + "terra.bdbag", + "terra.pfb", + "curl", + "verbatim.jsonl" + ] + }, + "description": "\nThe desired format of the output.\n\n- `compact` (the default) for a compact,\n tab-separated manifest\n\n- `terra.bdbag` for a manifest in the\n [BDBag format][1]. This provides a ZIP file containing two\n manifests: one for Participants (aka Donors) and one for\n Samples (aka Specimens). For more on the format of the\n manifests see [documentation here][2].\n\n- `terra.pfb` for a manifest in the [PFB\n format][3]. This format is mainly used for exporting data to\n Terra.\n\n- `curl` for a [curl configuration\n file][4] manifest. This manifest can be used with the curl\n program to download all the files listed in the manifest.\n\n- `verbatim.jsonl` for a verbatim\n manifest in [JSONL][5] format. Each line contains an\n unaltered metadata entity from the underlying repository.\n\n[1]: https://bd2k.ini.usc.edu/tools/bdbag/\n\n[2]: https://software.broadinstitute.org/firecloud/documentation/article?id=10954\n\n[3]: https://github.com/uc-cdis/pypfb\n\n[4]: https://curl.haxx.se/docs/manpage.html#-K\n\n[5]: https://jsonlines.org/\n" + } + ], + "responses": { + "200": { + "description": "\nWhen handling this response, clients should wait the\nnumber of seconds given in the `Retry-After` property of\nthe response body and then make another XHR request to\nthe URL specified in the `Location` property.\n\nFor a detailed description of these properties see the\ndocumentation for the respective response headers\ndocumented under \n[POST /manifest/files][1].\n\n[1]: #operations-Manifests-post_manifest_files\n\n\nNote: For a 200 status code response whose body has the\n`Status` property set to 302, the `Location` property\nmay reference the [GET /manifest/files/{token}][2]\nendpoint and that endpoint may return yet another\nredirect, this time a genuine (not emulated) 302 status\nredirect to the actual location of the manifest.\n\n[2]: #operations-Manifests-get_manifest_files__token_\n\nNote: A 200 status response with a `Status` property of\n302 in its body additionally contains a `CommandLine`\nproperty that lists, for a number of commonly used\nshells, a command line suitable for downloading the\nmanifest.\n", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "Status": { + "type": "integer", + "format": "int64" + }, + "Location": { + "type": "string", + "format": "url" + }, + "Retry-After": { + "type": "integer", + "format": "int64" + }, + "CommandLine": { + "type": "object", + "properties": { + "cmd.exe": { + "type": "string" + }, + "bash": { + "type": "string" + } + }, + "required": [ + "cmd.exe", + "bash" + ], + "additionalProperties": false + } + }, + "required": [ + "Status", + "Location" + ], + "additionalProperties": false + } + } + } + } + } + }, "put": { "tags": [ "Manifests" @@ -10895,7 +13610,7 @@ ], "responses": { "200": { - "description": "\nWhen handling this response, clients should wait the\nnumber of seconds given in the `Retry-After` property of\nthe response body and then make another XHR request to\nthe URL specified in the `Location` property.\n\nFor a detailed description of these properties see the\ndocumentation for the respective response headers\ndocumented under \n[PUT /manifest/files][1].\n\n[1]: #operations-Manifests-put_manifest_files\n\n\nNote: For a 200 status code response whose body has the\n`Status` property set to 302, the `Location` property\nmay reference the [GET /manifest/files/{token}][2]\nendpoint and that endpoint may return yet another\nredirect, this time a genuine (not emulated) 302 status\nredirect to the actual location of the manifest.\n\n[2]: #operations-Manifests-get_manifest_files\n\nNote: A 200 status response with a `Status` property of\n302 in its body additionally contains a `CommandLine`\nproperty that lists, for a number of commonly used\nshells, a command line suitable for downloading the\nmanifest.\n", + "description": "\nWhen handling this response, clients should wait the\nnumber of seconds given in the `Retry-After` property of\nthe response body and then make another XHR request to\nthe URL specified in the `Location` property.\n\nFor a detailed description of these properties see the\ndocumentation for the respective response headers\ndocumented under \n[PUT /manifest/files][1].\n\n[1]: #operations-Manifests-put_manifest_files\n\n\nNote: For a 200 status code response whose body has the\n`Status` property set to 302, the `Location` property\nmay reference the [GET /manifest/files/{token}][2]\nendpoint and that endpoint may return yet another\nredirect, this time a genuine (not emulated) 302 status\nredirect to the actual location of the manifest.\n\n[2]: #operations-Manifests-get_manifest_files__token_\n\nNote: A 200 status response with a `Status` property of\n302 in its body additionally contains a `CommandLine`\nproperty that lists, for a number of commonly used\nshells, a command line suitable for downloading the\nmanifest.\n", "content": { "application/json": { "schema": { @@ -10959,11 +13674,11 @@ "Manifests" ], "summary": "Determine status of a manifest preparation job via XHR", - "description": "\nCheck on the status of an ongoing manifest preparation job,\nreturning a 200 status response whose JSON body emulates the\nHTTP headers that would be found in a response to an equivalent\nrequest to the [GET /manifest/files/{token}][1] endpoint.\n\nWhenever client-side JavaScript code is used in a web\napplication to request the preparation of a manifest from Azul,\nthis endpoint should be used instead of [GET\n/manifest/files/{token}][1]. This way, the client can use XHR to\nmake the request, retaining full control over the handling of\nredirects and enabling the client to bypass certain limitations\non the native handling of redirects in web browsers. For\nexample, most browsers ignore the `Retry-After` header in\nredirect responses, causing them to prematurely exhaust the\nupper limit on the number of consecutive redirects, before the\nmanifest generation job is done.\n\n[1]: #operations-Manifests-get_manifest_files\n", + "description": "\nCheck on the status of an ongoing manifest preparation job,\nreturning a 200 status response whose JSON body emulates the\nHTTP headers that would be found in a response to an equivalent\nrequest to the [GET /manifest/files/{token}][1] endpoint.\n\nWhenever client-side JavaScript code is used in a web\napplication to request the preparation of a manifest from Azul,\nthis endpoint should be used instead of [GET\n/manifest/files/{token}][1]. This way, the client can use XHR to\nmake the request, retaining full control over the handling of\nredirects and enabling the client to bypass certain limitations\non the native handling of redirects in web browsers. For\nexample, most browsers ignore the `Retry-After` header in\nredirect responses, causing them to prematurely exhaust the\nupper limit on the number of consecutive redirects, before the\nmanifest generation job is done.\n\n[1]: #operations-Manifests-get_manifest_files__token_\n", "parameters": [], "responses": { "200": { - "description": "\nWhen handling this response, clients should wait the\nnumber of seconds given in the `Retry-After` property of\nthe response body and then make another XHR request to\nthe URL specified in the `Location` property.\n\nFor a detailed description of these properties see the\ndocumentation for the respective response headers\ndocumented under \n[GET /manifest/files/{token}][1].\n\n[1]: #operations-Manifests-get_manifest_files\n\n\nNote: For a 200 status code response whose body has the\n`Status` property set to 302, the `Location` property\nmay reference the [GET /manifest/files/{token}][2]\nendpoint and that endpoint may return yet another\nredirect, this time a genuine (not emulated) 302 status\nredirect to the actual location of the manifest.\n\n[2]: #operations-Manifests-get_manifest_files\n\nNote: A 200 status response with a `Status` property of\n302 in its body additionally contains a `CommandLine`\nproperty that lists, for a number of commonly used\nshells, a command line suitable for downloading the\nmanifest.\n", + "description": "\nWhen handling this response, clients should wait the\nnumber of seconds given in the `Retry-After` property of\nthe response body and then make another XHR request to\nthe URL specified in the `Location` property.\n\nFor a detailed description of these properties see the\ndocumentation for the respective response headers\ndocumented under \n[GET /manifest/files/{token}][1].\n\n[1]: #operations-Manifests-get_manifest_files__token_\n\n\nNote: For a 200 status code response whose body has the\n`Status` property set to 302, the `Location` property\nmay reference the [GET /manifest/files/{token}][2]\nendpoint and that endpoint may return yet another\nredirect, this time a genuine (not emulated) 302 status\nredirect to the actual location of the manifest.\n\n[2]: #operations-Manifests-get_manifest_files__token_\n\nNote: A 200 status response with a `Status` property of\n302 in its body additionally contains a `CommandLine`\nproperty that lists, for a number of commonly used\nshells, a command line suitable for downloading the\nmanifest.\n", "content": { "application/json": { "schema": { diff --git a/test/integration_test.py b/test/integration_test.py index 9870af544f..b6400db57f 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -587,60 +587,68 @@ def _test_manifest(self, catalog: CatalogName): ManifestFormat.verbatim_jsonl: self._check_jsonl_manifest } for format in [None, *supported_formats]: - # IT catalogs with just one public source are always indexed - # completely if that source contains less than the minimum number of - # bundles required. So regardless of any randomness employed by this - # test, manifests derived from these catalogs will always be based - # on the same content hash. Since the resulting reuse of cached - # manifests interferes with this test, we need another means of - # randomizing the manifest key: a random but all-inclusive filter. - tibi_byte = 1024 ** 4 - filters = { - self._file_size_facet(catalog): { - 'within': [[0, tibi_byte + self.random.randint(0, tibi_byte)]] + for initial_method in [PUT, POST]: + # IT catalogs with just one public source are always indexed + # completely if that source contains less than the minimum number of + # bundles required. So regardless of any randomness employed by this + # test, manifests derived from these catalogs will always be based + # on the same content hash. Since the resulting reuse of cached + # manifests interferes with this test, we need another means of + # randomizing the manifest key: a random but all-inclusive filter. + tibi_byte = 1024 ** 4 + filters = { + self._file_size_facet(catalog): { + 'within': [[0, tibi_byte + self.random.randint(0, tibi_byte)]] + } } - } - first_fetch = bool(self.random.getrandbits(1)) - for fetch in [first_fetch, not first_fetch]: - with self.subTest('manifest', catalog=catalog, format=format, fetch=fetch): - args = dict(catalog=catalog, filters=json.dumps(filters)) - if format is None: - validator = validators[first(supported_formats)] - else: - validator = validators[format] - args['format'] = format.value - - # Wrap self._get_url to collect all HTTP responses - _get_url = self._get_url - responses = list() - - def get_url(*args, **kwargs): - response = _get_url(*args, **kwargs) - responses.append(response) - return response - - with mock.patch.object(self, '_get_url', new=get_url): - - # Make multiple identical concurrent requests to test - # the idempotence of manifest generation, and its - # resilience against DOS attacks. - - def worker(_): - response = self._check_endpoint(PUT, '/manifest/files', args=args, fetch=fetch) - validator(catalog, response) - - num_workers = 3 - with ThreadPoolExecutor(max_workers=num_workers) as tpe: - results = list(tpe.map(worker, range(num_workers))) - - self.assertEqual([None] * num_workers, results) - execution_ids = self._manifest_execution_ids(responses, fetch=fetch) - # The second iteration of the inner-most loop re-requests - # the manifest with only `fetch` being different. In that - # case, the manifest will already be cached and no step - # function execution is expected to have been started. - expect_execution = fetch == first_fetch - self.assertEqual(1 if expect_execution else 0, len(execution_ids)) + first_fetch = bool(self.random.getrandbits(1)) + for fetch in [first_fetch, not first_fetch]: + with self.subTest('manifest', + catalog=catalog, + format=format, + initial_method=initial_method, + fetch=fetch): + args = dict(catalog=catalog, filters=json.dumps(filters)) + if format is None: + validator = validators[first(supported_formats)] + else: + validator = validators[format] + args['format'] = format.value + + # Wrap self._get_url to collect all HTTP responses + _get_url = self._get_url + responses = list() + + def get_url(*args, **kwargs): + response = _get_url(*args, **kwargs) + responses.append(response) + return response + + with mock.patch.object(self, '_get_url', new=get_url): + + # Make multiple identical concurrent requests to test + # the idempotence of manifest generation, and its + # resilience against DOS attacks. + + def worker(_): + response = self._check_endpoint(initial_method, + '/manifest/files', + args=args, + fetch=fetch) + validator(catalog, response) + + num_workers = 3 + with ThreadPoolExecutor(max_workers=num_workers) as tpe: + results = list(tpe.map(worker, range(num_workers))) + + self.assertEqual([None] * num_workers, results) + execution_ids = self._manifest_execution_ids(responses, fetch=fetch) + # The second iteration of the inner-most loop re-requests + # the manifest with only `fetch` being different. In that + # case, the manifest will already be cached and no step + # function execution is expected to have been started. + expect_execution = fetch == first_fetch + self.assertEqual(1 if expect_execution else 0, len(execution_ids)) def _manifest_execution_ids(self, responses: list[urllib3.HTTPResponse], diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index 86577cdbd6..1afa6d77bb 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -1502,7 +1502,12 @@ def test_manifest(self, Verify the response from manifest endpoints for all manifest formats """ - def test(*, format: ManifestFormat, fetch: bool, url: Optional[furl] = None): + def test(*, + format: ManifestFormat, + fetch: bool, + initial_method: Optional[str] = None, + url: Optional[furl] = None): + assert initial_method or url, (format, fetch) object_url = furl('https://url.to.manifest?foo=bar') default_file_name = 'some_object_key.csv' manifest_key = ManifestKey(catalog=self.catalog, @@ -1549,7 +1554,7 @@ def test(*, format: ManifestFormat, fetch: bool, url: Optional[furl] = None): 'bash': f'curl {options} {file_name} {expected_url_for_bash}' } if url is None: - method, request_url = 'PUT', self.base_url.set(path=path, args=args) + method, request_url = initial_method, self.base_url.set(path=path, args=args) else: assert not fetch method, request_url = 'GET', url @@ -1560,7 +1565,7 @@ def test(*, format: ManifestFormat, fetch: bool, url: Optional[furl] = None): 'Location': str(expected_url), 'CommandLine': expected } - response = requests.request('PUT', str(request_url)) + response = requests.request(initial_method, str(request_url)) self.assertEqual(200, response.status_code) self.assertEqual(expected, response.json()) self.assertEqual('application/json', response.headers['Content-Type']) @@ -1579,8 +1584,9 @@ def test(*, format: ManifestFormat, fetch: bool, url: Optional[furl] = None): for format in self.app_module.app.metadata_plugin.manifest_formats: for fetch in True, False: - with self.subTest(format=format, fetch=fetch): - test(format=format, fetch=fetch) + for initial_method in ['PUT', 'POST']: + with self.subTest(format=format, fetch=fetch, initial_method=initial_method): + test(format=format, fetch=fetch, initial_method=initial_method) class TestManifestExpiration(AzulUnitTestCase):