diff --git a/poetry.lock b/poetry.lock index 203239b7a..798291b39 100644 --- a/poetry.lock +++ b/poetry.lock @@ -572,17 +572,6 @@ files = [ click = ">=4.0" PyYAML = ">=3.11" -[[package]] -name = "cloudpickle" -version = "3.0.0" -description = "Pickler class to extend the standard pickle.Pickler functionality" -optional = false -python-versions = ">=3.8" -files = [ - {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"}, - {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"}, -] - [[package]] name = "colorama" version = "0.4.6" @@ -2078,20 +2067,6 @@ files = [ {file = "lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d"}, ] -[[package]] -name = "loky" -version = "3.0.0" -description = "A robust implementation of concurrent.futures.ProcessPoolExecutor" -optional = false -python-versions = "*" -files = [ - {file = "loky-3.0.0-py2.py3-none-any.whl", hash = "sha256:1d5a4d778c7ff09c919aa3fbf2d879a2c7ac936a545c615af40e080a1c902b82"}, - {file = "loky-3.0.0.tar.gz", hash = "sha256:fd8750b24b283a579bafaf0631d114aa4487c682aef6fce01fa3635336297fdf"}, -] - -[package.dependencies] -cloudpickle = "*" - [[package]] name = "makefun" version = "1.15.2" @@ -3400,7 +3375,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3408,16 +3382,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3434,7 +3400,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3442,7 +3407,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -4379,13 +4343,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.2.0" +version = "4.3.1" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.2.0-py3-none-any.whl", hash = "sha256:ab5bc9c2bf5b90f271f1a9478eff7e9fca3e573578401ac706383ddb984d7a13"}, - {file = "synapseclient-4.2.0.tar.gz", hash = "sha256:89222661125de1795b1a096cf8c58b8115c19d6b0fa5846ed2a41cdb394ef773"}, + {file = "synapseclient-4.3.1-py3-none-any.whl", hash = "sha256:515fff80092c4acee010e272ae313533ae31f7cbe0a590f540f98fd10a18177b"}, + {file = "synapseclient-4.3.1.tar.gz", hash = "sha256:9d1c2cd1d6fe4fabb386290c0eed20944ab7e44e6713db40f19cf28babe3be3c"}, ] [package.dependencies] @@ -4393,7 +4357,6 @@ async-lru = ">=2.0.4,<2.1.0" asyncio-atexit = ">=1.0.1,<1.1.0" deprecated = ">=1.2.4,<2.0" httpx = ">=0.27.0,<0.28.0" -loky = ">=3.0.0,<3.1.0" nest-asyncio = ">=1.6.0,<1.7.0" opentelemetry-api = ">=1.21.0,<1.22.0" opentelemetry-exporter-otlp-proto-http = ">=1.21.0,<1.22.0" @@ -4405,11 +4368,11 @@ urllib3 = ">=1.26.18,<2" [package.extras] boto3 = ["boto3 (>=1.7.0,<2.0)"] -dev = ["black", "flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pre-commit", "pytest (>=6.0.0,<7.0)", "pytest-asyncio (>=0.19,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] +dev = ["black", "flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pre-commit", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] docs = ["markdown-include (>=0.8.1,<0.9.0)", "mkdocs (>=1.5.3)", "mkdocs-material (>=9.4.14)", "mkdocs-open-in-new-tab (>=1.0.3,<1.1.0)", "mkdocstrings (>=0.24.0)", "mkdocstrings-python (>=1.7.5)", "termynal (>=0.11.1)"] pandas = ["pandas (>=1.5,<3.0)"] pysftp = ["pysftp (>=0.2.8,<0.3)"] -tests = ["flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pytest (>=6.0.0,<7.0)", "pytest-asyncio (>=0.19,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] +tests = ["flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] [[package]] name = "tabulate" @@ -4944,4 +4907,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "5bf0c831977694ea541db24481181ec1980ec9589a2adbd9f30ed0fe7f2b2742" +content-hash = "e327e774115908a3edf6c70ff34869b026e560cf9262470b17ebe06359aac06a" diff --git a/pyproject.toml b/pyproject.toml index 3c2795140..92cf0d962 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "^4.1.0" +synapseclient = "4.3.1" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" diff --git a/schematic_api/Dockerfile b/schematic_api/Dockerfile index 15e979dee..cfcc6cab7 100644 --- a/schematic_api/Dockerfile +++ b/schematic_api/Dockerfile @@ -12,7 +12,7 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_NO_CACHE_DIR=off \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=200 \ - POETRY_VERSION=1.3.0 \ + POETRY_VERSION=1.7.1 \ APP_PARENT_DIR=/app \ NGINX_CONFIG=/etc/nginx/conf.d \ APP_DIR=/app/app \ diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index b2439133e..ae73428eb 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -1,63 +1,55 @@ +import json +import logging import os -from json.decoder import JSONDecodeError +import pathlib +import pickle import shutil import tempfile -import shutil +import time import urllib.request -import logging -import pathlib -import pickle +from functools import wraps +from json.decoder import JSONDecodeError +from typing import Any, List, Optional import connexion +import pandas as pd from connexion.decorators.uri_parsing import Swagger2URIParser -from werkzeug.debug import DebuggedApplication - -from flask_cors import cross_origin -from flask import send_from_directory from flask import current_app as app -from flask import request - -import pandas as pd -import json -from typing import Optional, List, Any -from functools import wraps - +from flask import request, send_from_directory +from flask_cors import cross_origin from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, ConsoleSpanExporter, + SimpleSpanProcessor, Span, ) -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter - -from schematic.configuration.configuration import CONFIG -from schematic.visualization.attributes_explorer import AttributesExplorer -from schematic.visualization.tangled_tree import TangledTree -from schematic.manifest.generator import ManifestGenerator -from schematic.models.metadata import MetadataModel - -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer - -from schematic.store.synapse import SynapseStorage, ManifestDownload +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF from synapseclient.core.exceptions import ( - SynapseHTTPError, SynapseAuthenticationError, - SynapseUnmetAccessRestrictions, + SynapseHTTPError, SynapseNoCredentialsError, SynapseTimeoutError, + SynapseUnmetAccessRestrictions, ) +from werkzeug.debug import DebuggedApplication + +from schematic.configuration.configuration import CONFIG +from schematic.manifest.generator import ManifestGenerator +from schematic.models.metadata import MetadataModel +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser +from schematic.store.synapse import ManifestDownload, SynapseStorage from schematic.utils.general import entity_type_mapping from schematic.utils.schema_utils import ( - get_property_label_from_display_name, DisplayLabelType, -) -from schematic.utils.schema_utils import ( get_property_label_from_display_name, - DisplayLabelType, ) +from schematic.visualization.attributes_explorer import AttributesExplorer +from schematic.visualization.tangled_tree import TangledTree logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) @@ -83,9 +75,24 @@ def export(self, spans: List[Span]) -> None: f.write(span_json_one_line) -trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) -# processor = SimpleSpanProcessor(FileSpanExporter("otel_spans_schemati_api.json")) -# trace.get_tracer_provider().add_span_processor(processor) +def set_up_tracing() -> None: + """Set up tracing for the API.""" + tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) + if tracing_export == "otlp": + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) + ) + elif tracing_export == "file": + timestamp_millis = int(time.time() * 1000) + file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson" + file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name) + processor = SimpleSpanProcessor(FileSpanExporter(file_path)) + trace.get_tracer_provider().add_span_processor(processor) + else: + trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) + + +set_up_tracing() tracer = trace.get_tracer("Schematic")