diff --git a/src/psqlgml/dictionaries/readers.py b/src/psqlgml/dictionaries/readers.py index 0548c7b..dc8d44c 100644 --- a/src/psqlgml/dictionaries/readers.py +++ b/src/psqlgml/dictionaries/readers.py @@ -1,7 +1,7 @@ import logging import os from pathlib import Path -from typing import Optional, cast +from typing import Optional, Union, cast from psqlgml.dictionaries import repository, schemas @@ -25,7 +25,10 @@ def __init__(self, name: str, version: str) -> None: self.reader: Optional[repository.Repository] = None - def local(self, base_directory: Optional[Path] = None) -> "DictionaryReader": + def local(self, base_directory: Optional[Union[str, Path]] = None) -> "DictionaryReader": + base_directory = ( + Path(base_directory) if isinstance(base_directory, str) else base_directory + ) logger.debug(f"Reading local Dictionary {self.name}: {self.version} @ {base_directory}") self._base_dir = base_directory or self._base_dir return self diff --git a/src/psqlgml/dictionaries/repository.py b/src/psqlgml/dictionaries/repository.py index c69e359..1ac3de2 100644 --- a/src/psqlgml/dictionaries/repository.py +++ b/src/psqlgml/dictionaries/repository.py @@ -26,7 +26,6 @@ def get_dictionary_directory(self, version: str) -> Path: @abc.abstractmethod def read(self, version: str) -> schemas.Dictionary: """Reads the specified dictionary version from the repository""" - ... @attr.s(auto_attribs=True) @@ -129,7 +128,7 @@ def get_commit_id(self, commit_ref: str) -> bytes: return obj.id if isinstance(obj, objects.Tag): return obj.object[1] - raise ValueError(f"Unrecognized commit {commit_ref}") + raise IOError(f"unknown commit type for ref {commit_ref}") def clone(self) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 85f3bec..7b62322 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,15 @@ +import glob +import shutil +import sys +import threading from pathlib import Path import pkg_resources import pytest +from _pytest.fixtures import SubRequest +from _pytest.legacypath import TempdirFactory +from dulwich import porcelain +from dulwich.server import DictBackend, TCPGitServer import psqlgml from tests.helpers import SchemaInfo @@ -32,3 +40,38 @@ def test_schema(local_schema: SchemaInfo) -> psqlgml.GmlSchema: version=local_schema.version, schema_location=local_schema.source_dir, ) + + +@pytest.fixture(scope="session") +def git_server(request: SubRequest, tmpdir_factory: TempdirFactory, data_dir: str): + + base_dir = f"{tmpdir_factory.mktemp('gitdata')}/rsvr" + + # copy dictionary files to tmp dir + shutil.copytree(f"{data_dir}/dictionary/0.1.0", f"{base_dir}/gdcdictionary/schemas") + + # make a git repo for tests + repo = porcelain.Repo.init(base_dir) + + # stage & commit dictionary files + paths = [p.replace(f"{base_dir}/", "") for p in glob.glob(f"{base_dir}/**", recursive=True)] + repo.stage(paths) + print(",".join([f.decode(sys.getfilesystemencoding()) for f in repo.open_index()])) + commit = repo.do_commit(b"Add items", committer=b"pytest ") + + # add tag and master branch + assert repo.head() == commit + repo.refs[b"refs/heads/master"] = commit + repo.refs[b"refs/tags/0.1.0"] = commit + + print("================ refs ====================") + print(repo.refs) + + # create remote repo + backend = DictBackend({b"/rsvr": repo}) + dul_server = TCPGitServer(backend, b"localhost", 0) + threading.Thread(target=dul_server.serve).start() + + server_addr, server_port = dul_server.socket.getsockname() + request.addfinalizer(dul_server.shutdown) + return f"git://{server_addr}:{server_port}/rsvr" diff --git a/tests/data/dictionary/0.1.0/samples/project.yaml b/tests/data/dictionary/0.1.0/samples/project.yaml new file mode 100644 index 0000000..b4c557f --- /dev/null +++ b/tests/data/dictionary/0.1.0/samples/project.yaml @@ -0,0 +1 @@ +id: projects diff --git a/tests/unit/test_git_repository.py b/tests/unit/test_git_repository.py index ab4719f..28a37d1 100644 --- a/tests/unit/test_git_repository.py +++ b/tests/unit/test_git_repository.py @@ -7,8 +7,6 @@ from psqlgml.dictionaries import repository -REMOTE_GIT_URL = "https://github.com/NCI-GDC/gdcdictionary.git" - @pytest.mark.parametrize( "default_base, expectation", @@ -23,7 +21,7 @@ def test_get_dictionary_dir(data_dir: str, default_base: str, expectation) -> No gml_dir = f"{Path.home()}/.gml/dictionaries" if default_base else data_dir with mock.patch.dict(os.environ, {"GML_DICTIONARY_HOME": gml_dir}): - repo = repository.GitRepository(name="dictionary", url=REMOTE_GIT_URL, lazy_load=True) + repo = repository.GitRepository(name="dictionary", url="", lazy_load=True) assert repo.name == "dictionary" assert Path(expectation) == repo.get_dictionary_directory("0.1.0") @@ -40,14 +38,14 @@ def test_get_local_git_dir(local_git_home: str) -> None: with mock.patch.dict(os.environ, {"GML_GIT_HOME": local_git_home}): - repo = repository.GitRepository(name="dictionary", url=REMOTE_GIT_URL, lazy_load=True) + repo = repository.GitRepository(name="dictionary", url="", lazy_load=True) assert repo.name == "dictionary" assert Path(f"{local_git_home}/dictionary") == repo.local_directory def test_lazy_load_no_clone(tmpdir: Path) -> None: with mock.patch.dict(os.environ, {"GML_GIT_HOME": str(tmpdir)}): - rm = repository.GitRepository(url=REMOTE_GIT_URL, name="smiths", lazy_load=True) + rm = repository.GitRepository(url="", name="smiths", lazy_load=True) assert rm.is_cloned is False @@ -55,7 +53,71 @@ def test_lazy_load_no_clone(tmpdir: Path) -> None: "is_tag, expected_ref", [(True, "refs/tags/0.1.0"), (False, "refs/remotes/origin/0.1.0")] ) def test_get_commit_ref(is_tag: bool, expected_ref: str) -> None: - rm = repository.GitRepository( - url=REMOTE_GIT_URL, name="smiths", lazy_load=True, is_tag=is_tag - ) + rm = repository.GitRepository(url="", name="smiths", lazy_load=True, is_tag=is_tag) assert expected_ref == rm.get_commit_ref("0.1.0") + + +@pytest.mark.parametrize( + "commit, is_tag", + [ + ("0.1.0", True), + ("master", False), + ], +) +def test_get_git_commit_id(tmpdir: Path, git_server: str, commit: str, is_tag: bool) -> None: + with mock.patch.dict(os.environ, {"GML_GIT_HOME": str(tmpdir)}): + rm = repository.GitRepository(url=git_server, name="smiths", is_tag=is_tag) + rm.clone() + assert rm.get_commit_id(rm.get_commit_ref(commit)) + + +def test_get_invalid_commit_id(tmpdir: Path, git_server: str) -> None: + with mock.patch.dict(os.environ, {"GML_GIT_HOME": str(tmpdir)}): + rm = repository.GitRepository(url=git_server, name="smiths", is_tag=False) + rm.clone() + assert rm.get_commit_id(rm.get_commit_ref("master")) + + +@pytest.mark.parametrize("lazy", [True, False]) +def test_read_remote_dictionary(tmpdir: Path, git_server: str, lazy: bool) -> None: + with mock.patch.dict(os.environ, {"GML_GIT_HOME": str(tmpdir)}): + + project = repository.GitRepository( + url=git_server, + name="smiths", + force=True, + schema_path="gdcdictionary/schemas", + lazy_load=lazy, + ) + chk_dir = project.get_dictionary_directory("0.1.0") + dictionary = project.read("0.1.0") + assert chk_dir.exists() + + entries = [f.name for f in chk_dir.iterdir()] + assert "program.yaml" in entries + + assert dictionary.name == "smiths" + assert dictionary.version == "0.1.0" + + +def test_read_existing_remote_dictionary(tmpdir: Path, git_server: str) -> None: + with mock.patch.dict(os.environ, {"GML_GIT_HOME": str(tmpdir)}): + + project = repository.GitRepository( + url=git_server, + name="smiths", + force=True, + schema_path="gdcdictionary/schemas", + ) + project.read("0.1.0") + + p2 = repository.GitRepository( + url=git_server, + name="smiths", + force=False, + schema_path="gdcdictionary/schemas", + ) + dictionary = p2.read("0.1.0") + + assert dictionary.name == "smiths" + assert dictionary.version == "0.1.0" diff --git a/tests/unit/test_readers.py b/tests/unit/test_readers.py new file mode 100644 index 0000000..fcf0299 --- /dev/null +++ b/tests/unit/test_readers.py @@ -0,0 +1,37 @@ +import os +from pathlib import Path +from unittest import mock + +from psqlgml.dictionaries import readers + + +def test_local_reader(data_dir: str): + dictionary = readers.DictionaryReader("dictionary", "0.1.0").local(f"{data_dir}").read() + assert dictionary.name == "dictionary" + assert dictionary.version == "0.1.0" + + +def test_load_local(data_dir: str): + dictionary = readers.load_local("dictionary", "0.1.0", data_dir) + assert dictionary.name == "dictionary" + assert dictionary.version == "0.1.0" + + +def test_remote_reader(data_dir: str, git_server: str, tmpdir: Path) -> None: + with mock.patch.dict( + os.environ, + {"GML_GIT_HOME": f"{tmpdir}/git", "GML_DICTIONARY_HOME": f"{tmpdir}/dictionaries"}, + ): + dictionary = readers.DictionaryReader("dictionary", "0.1.0").git(git_server, True).read() + assert dictionary.name == "dictionary" + assert dictionary.version == "0.1.0" + + +def test_load_remote(data_dir: str, git_server: str, tmpdir: Path) -> None: + with mock.patch.dict( + os.environ, + {"GML_GIT_HOME": f"{tmpdir}/git", "GML_DICTIONARY_HOME": f"{tmpdir}/dictionaries"}, + ): + dictionary = readers.load(name="dictionary", version="0.1.0", git_url=git_server) + assert dictionary.name == "dictionary" + assert dictionary.version == "0.1.0"