From 18e5e64eb0f378f932c37980ffa9851c346d12a2 Mon Sep 17 00:00:00 2001 From: Rowland Ogwara Date: Thu, 2 Feb 2023 16:24:50 -0600 Subject: [PATCH 1/4] DEV-1396: add gdcdatamodel2 dependency and bump dictionary --- dev-requirements.txt | 10 +++++----- esbuild/__init__.py | 3 +++ requirements.txt | 41 +++++++++++++++-------------------------- setup.py | 7 ++++--- tests/__init__.py | 3 +++ 5 files changed, 30 insertions(+), 34 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 151f0202..2d8c606b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.6 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile dev-requirements.in @@ -91,12 +91,12 @@ pyrsistent==0.17.3 # via # -c requirements.txt # jsonschema +pytest-cov==2.10.1 + # via -r dev-requirements.in pytest==7.0.1 # via # -r dev-requirements.in # pytest-cov -pytest-cov==2.10.1 - # via -r dev-requirements.in python-dateutil==2.8.0 # via -r dev-requirements.in requests==2.25.1 @@ -112,13 +112,13 @@ six==1.15.0 # mock # python-dateutil # sqlalchemy-utils +sqlalchemy-utils==0.36.3 + # via indexd sqlalchemy==1.3.3 # via # -c requirements.txt # indexd # sqlalchemy-utils -sqlalchemy-utils==0.36.3 - # via indexd tomli==1.2.3 # via pytest urllib3==1.26.2 diff --git a/esbuild/__init__.py b/esbuild/__init__.py index e69de29b..54a78627 100644 --- a/esbuild/__init__.py +++ b/esbuild/__init__.py @@ -0,0 +1,3 @@ +import os + +os.environ["GDCDICTIONARY_NAMESPACE"] = "gdc2" diff --git a/requirements.txt b/requirements.txt index 72f7df8b..60dba693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.6 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile @@ -36,41 +36,33 @@ elasticsearch==7.10.1 # gdcmodels future==0.18.2 # via esbuild (setup.py) -gdc-ng-models @ git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2 - # via - # esbuild (setup.py) - # gdcdatamodel -gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.4.0 +git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models # via esbuild (setup.py) -gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.4.0 - # via - # esbuild (setup.py) - # gdcdatamodel -gdcmodels @ git+ssh://git@github.com/NCI-GDC/gdc-models.git@2.8.1-rc.2 +git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2 + # via esbuild (setup.py) +git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary + # via esbuild (setup.py) +git+ssh://git@github.com/NCI-GDC/gdc-models.git@2.8.1-rc.2#egg=gdcmodels # via # esbuild (setup.py) # normalizer -gitdb==4.0.5 - # via gitdb2 gitdb2==4.0.2 # via gitpython +gitdb==4.0.5 + # via gitdb2 gitpython==2.1.11 # via normalizer -graphviz==0.4.10 - # via gdcdatamodel idna==2.10 # via requests importlib-metadata==2.1.1 # via jsonschema -indexclient @ git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4 +git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4#egg=indexclient # via esbuild (setup.py) jsonschema==3.2.0 - # via - # gdcdatamodel - # gdcdictionary + # via gdcdictionary networkx==2.4 # via esbuild (setup.py) -normalizer @ git+ssh://git@github.com/NCI-GDC/normalizer.git@2.0.4 +git+ssh://git@github.com/NCI-GDC/normalizer.git@2.0.4#egg=normalizer # via esbuild (setup.py) ordered-set==4.0.2 # via deepdiff @@ -78,10 +70,10 @@ pika==1.1.0 # via queueclient progressbar2==3.53.1 # via esbuild (setup.py) -psqlgraph @ git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.3.0 +git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.4.2#egg=psqlgraph # via # esbuild (setup.py) - # gdcdatamodel + # gdcdatamodel2 psycopg2==2.8.6 # via psqlgraph pyrsistent==0.17.3 @@ -90,14 +82,12 @@ python-dotenv==0.10.5 # via esbuild (setup.py) python-utils==2.4.0 # via progressbar2 -pytz==2020.4 - # via gdcdatamodel pyyaml==5.3.1 # via # gdcdictionary # gdcmodels # normalizer -queueclient @ git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0 +git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0#egg=queueclient # via esbuild (setup.py) requests==2.25.1 # via @@ -114,7 +104,6 @@ six==1.15.0 # gdcmodels # jsonschema # progressbar2 - # psqlgraph # python-utils smmap==3.0.4 # via gitdb diff --git a/setup.py b/setup.py index 656c5bc0..04c0bf24 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,10 @@ "python-dotenv~=0.10.3", "requests~=2.7", "SQLAlchemy==1.3.3", - "psqlgraph @ git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.3.0#egg=psqlgraph", - "gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.4.0#egg=gdcdictionary", - "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.4.0#egg=gdcdatamodel", + "psqlgraph @ git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.4.2#egg=psqlgraph", + "gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary", + "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.6.5#egg=gdcdatamodel", + "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2", "gdc_ng_models @ git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models", "indexclient @ git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4#egg=indexclient", "queueclient @ git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0#egg=queueclient", diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..54a78627 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import os + +os.environ["GDCDICTIONARY_NAMESPACE"] = "gdc2" From 71844de8f984178783ab0499bb7b46f5c1a48524 Mon Sep 17 00:00:00 2001 From: Rowland Ogwara Date: Thu, 2 Feb 2023 17:00:13 -0600 Subject: [PATCH 2/4] fix gdcdatamodel typo --- dev-requirements.txt | 29 +++++++++++++++---------- requirements.txt => requirements.txt.bf | 0 setup.py | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) rename requirements.txt => requirements.txt.bf (100%) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2d8c606b..9e51650c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,20 +14,20 @@ # indexd -e git+ssh://git@github.com/NCI-GDC/indexd.git@2.9.0-rc.1#egg=indexd # via -r dev-requirements.in -attrs==20.3.0 +attrs==22.2.0 # via # -c requirements.txt # jsonschema # pytest -cdislogging==1.0.0 +cdislogging==1.1.1 # via # -c requirements.txt # indexd -certifi==2020.12.5 +certifi==2022.12.7 # via # -c requirements.txt # requests -chardet==4.0.0 +charset-normalizer==2.0.12 # via # -c requirements.txt # requests @@ -41,15 +41,15 @@ decorator==4.4.2 # jsonpath-rw flask==1.1.2 # via indexd -future==0.18.2 +future==0.18.3 # via # -c requirements.txt # indexd -idna==2.10 +idna==3.4 # via # -c requirements.txt # requests -importlib-metadata==2.1.1 +importlib-metadata==4.8.3 # via # -c requirements.txt # jsonschema @@ -87,7 +87,7 @@ py==1.10.0 # via pytest pyparsing==2.4.7 # via packaging -pyrsistent==0.17.3 +pyrsistent==0.16.1 # via # -c requirements.txt # jsonschema @@ -99,17 +99,18 @@ pytest==7.0.1 # pytest-cov python-dateutil==2.8.0 # via -r dev-requirements.in -requests==2.25.1 +requests==2.27.1 # via # -c requirements.txt # doiclient # dosclient -six==1.15.0 +six==1.16.0 # via # -c requirements.txt # jsonpath-rw # jsonschema # mock + # pyrsistent # python-dateutil # sqlalchemy-utils sqlalchemy-utils==0.36.3 @@ -121,7 +122,11 @@ sqlalchemy==1.3.3 # sqlalchemy-utils tomli==1.2.3 # via pytest -urllib3==1.26.2 +typing-extensions==4.1.1 + # via + # -c requirements.txt + # importlib-metadata +urllib3==1.26.14 # via # -c requirements.txt # requests @@ -129,7 +134,7 @@ werkzeug==0.16.1 # via # flask # indexd -zipp==1.2.0 +zipp==3.6.0 # via # -c requirements.txt # importlib-metadata diff --git a/requirements.txt b/requirements.txt.bf similarity index 100% rename from requirements.txt rename to requirements.txt.bf diff --git a/setup.py b/setup.py index 04c0bf24..77641219 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ "psqlgraph @ git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.4.2#egg=psqlgraph", "gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary", "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.6.5#egg=gdcdatamodel", - "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2", + "gdcdatamodel2 @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2", "gdc_ng_models @ git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models", "indexclient @ git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4#egg=indexclient", "queueclient @ git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0#egg=queueclient", From 5d697b57926a64449e5620c47b67b4e310e02ee5 Mon Sep 17 00:00:00 2001 From: Rowland Ogwara Date: Fri, 3 Feb 2023 09:33:57 -0600 Subject: [PATCH 3/4] bump requirements --- requirements.txt.bf => requirements.txt | 67 ++++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) rename requirements.txt.bf => requirements.txt (75%) diff --git a/requirements.txt.bf b/requirements.txt similarity index 75% rename from requirements.txt.bf rename to requirements.txt index 60dba693..4845faa3 100644 --- a/requirements.txt.bf +++ b/requirements.txt @@ -6,60 +6,70 @@ # addict==2.1.3 # via esbuild (setup.py) -attrs==20.3.0 +attrs==22.2.0 # via jsonschema backports.tempfile==1.0 # via normalizer backports.weakref==1.0.post1 # via backports.tempfile -cdislogging==1.0.0 +cdislogging==1.1.1 # via esbuild (setup.py) -certifi==2020.12.5 +certifi==2022.12.7 # via # elasticsearch # requests -chardet==4.0.0 +charset-normalizer==2.0.12 # via requests -datadog==0.39.0 +datadog==0.43.0 # via esbuild (setup.py) decorator==4.4.2 # via - # datadog + # gdcdatamodel # networkx deepdiff==4.3.2 # via esbuild (setup.py) dictdiffer==0.8.1 # via esbuild (setup.py) -elasticsearch==7.10.1 +elasticsearch==7.17.8 # via # esbuild (setup.py) # gdcmodels -future==0.18.2 +future==0.18.3 # via esbuild (setup.py) git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models - # via esbuild (setup.py) + # via + # esbuild (setup.py) + # gdcdatamodel git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2 # via esbuild (setup.py) -git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary +git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.6.5#egg=gdcdatamodel # via esbuild (setup.py) +git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary + # via + # esbuild (setup.py) + # gdcdatamodel git+ssh://git@github.com/NCI-GDC/gdc-models.git@2.8.1-rc.2#egg=gdcmodels # via # esbuild (setup.py) # normalizer gitdb2==4.0.2 # via gitpython -gitdb==4.0.5 +gitdb==4.0.9 # via gitdb2 gitpython==2.1.11 # via normalizer -idna==2.10 +graphviz==0.16 + # via gdcdatamodel +idna==3.4 # via requests -importlib-metadata==2.1.1 +importlib-metadata==4.8.3 # via jsonschema git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4#egg=indexclient # via esbuild (setup.py) jsonschema==3.2.0 - # via gdcdictionary + # via + # gdcdatamodel + # gdcdictionary networkx==2.4 # via esbuild (setup.py) git+ssh://git@github.com/NCI-GDC/normalizer.git@2.0.4#egg=normalizer @@ -68,28 +78,33 @@ ordered-set==4.0.2 # via deepdiff pika==1.1.0 # via queueclient -progressbar2==3.53.1 +progressbar2==3.55.0 # via esbuild (setup.py) git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.4.2#egg=psqlgraph # via # esbuild (setup.py) + # gdcdatamodel # gdcdatamodel2 psycopg2==2.8.6 # via psqlgraph -pyrsistent==0.17.3 - # via jsonschema +pyrsistent==0.16.1 + # via + # gdcdatamodel + # jsonschema python-dotenv==0.10.5 # via esbuild (setup.py) -python-utils==2.4.0 +python-utils==3.4.5 # via progressbar2 -pyyaml==5.3.1 +pytz==2020.5 + # via gdcdatamodel +pyyaml==5.4.1 # via # gdcdictionary # gdcmodels # normalizer git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0#egg=queueclient # via esbuild (setup.py) -requests==2.25.1 +requests==2.27.1 # via # datadog # esbuild (setup.py) @@ -99,25 +114,27 @@ rstr==2.2.6 # via psqlgraph simplejson==3.16.0 # via queueclient -six==1.15.0 +six==1.16.0 # via # gdcmodels # jsonschema # progressbar2 - # python-utils -smmap==3.0.4 + # pyrsistent +smmap==5.0.0 # via gitdb sqlalchemy==1.3.3 # via # esbuild (setup.py) # psqlgraph -urllib3==1.26.2 +typing-extensions==4.1.1 + # via importlib-metadata +urllib3==1.26.14 # via # elasticsearch # requests xlocal==0.5 # via psqlgraph -zipp==1.2.0 +zipp==3.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 278a451eb1efa41623adb18abfd50fb335ed2509 Mon Sep 17 00:00:00 2001 From: Rowland Ogwara Date: Tue, 21 Feb 2023 17:18:04 -0600 Subject: [PATCH 4/4] rollback dictionary version --- esbuild/graph/common/mappings.py | 16 ++++++++-------- requirements.txt | 4 ++-- setup.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/esbuild/graph/common/mappings.py b/esbuild/graph/common/mappings.py index 74ceb8aa..cbe3328f 100644 --- a/esbuild/graph/common/mappings.py +++ b/esbuild/graph/common/mappings.py @@ -3,7 +3,7 @@ Common definitions for building GDC Elasticsearch mappings """ - +import typing as t from copy import deepcopy from addict import Dict @@ -313,17 +313,17 @@ def _get_header(cls, source): return header @classmethod - def get_base_properties(cls, source, include_id=True): + def get_base_properties(cls, node_label: str, include_id=True): # Get properties from schema - node_type = Node.get_subclass(source) - assert node_type, f"No model for {source}" + node_type = Node.get_subclass(node_label) + assert node_type, f"No model for {node_label}" - properties = dict(node_type.get_pg_properties()) + properties: t.Dict[str, t.Tuple[t.Type]] = dict(node_type.get_pg_properties()) doc = Dict() if include_id: # Add id to document - id_name = f"{source}_id" + id_name = f"{node_label}_id" doc[id_name] = STRING if properties.pop("submitter_id", None): @@ -335,12 +335,12 @@ def get_base_properties(cls, source, include_id=True): # assign the type doc[field] = {"type": _type} - if source == "project": + if node_label == "project": # Remove some fields from project document for key in cls.project_keys_to_hide: doc.pop(key) - if source != "project": + if node_label != "project": doc.pop("project_id", None) doc.pop("batch_id", None) doc.pop("file_state", None) diff --git a/requirements.txt b/requirements.txt index 4845faa3..f2e4f76f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,11 +40,11 @@ git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models # via # esbuild (setup.py) # gdcdatamodel -git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2 +git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.4.2#egg=gdcdatamodel2 # via esbuild (setup.py) git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.6.5#egg=gdcdatamodel # via esbuild (setup.py) -git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary +git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.4.2#egg=gdcdictionary # via # esbuild (setup.py) # gdcdatamodel diff --git a/setup.py b/setup.py index 77641219..46b465b8 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,9 @@ "requests~=2.7", "SQLAlchemy==1.3.3", "psqlgraph @ git+ssh://git@github.com/NCI-GDC/psqlgraph.git@3.4.2#egg=psqlgraph", - "gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.6.3#egg=gdcdictionary", + "gdcdictionary @ git+ssh://git@github.com/NCI-GDC/gdcdictionary.git@2.4.2#egg=gdcdictionary", "gdcdatamodel @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel.git@3.6.5#egg=gdcdatamodel", - "gdcdatamodel2 @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.6.3#egg=gdcdatamodel2", + "gdcdatamodel2 @ git+ssh://git@github.com/NCI-GDC/gdcdatamodel2.git@2.4.2#egg=gdcdatamodel2", "gdc_ng_models @ git+ssh://git@github.com/NCI-GDC/gdc-ng-models.git@1.5.2#egg=gdc_ng_models", "indexclient @ git+ssh://git@github.com/NCI-GDC/indexclient.git@2.1.0-rc.4#egg=indexclient", "queueclient @ git+ssh://git@github.com/NCI-GDC/queueclient.git@1.2.0#egg=queueclient",