From 87c749d118d3a8a463763738d06ec0d31aec3693 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Thu, 24 Feb 2022 16:40:11 +0000 Subject: [PATCH 01/10] schema validation added --- .github/workflows/tester.yaml | 2 + src/schema/dosdp_schema.yaml | 2 +- src/schema/schema_validator.py | 12 +++++ src/schema/test/generic_test/schema_test.py | 9 ++++ ...bnormallyHyperplasticAnatomicalEntity.yaml | 50 +++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/schema/schema_validator.py create mode 100644 src/schema/test/positive_test_set/patterns/abnormallyHyperplasticAnatomicalEntity.yaml diff --git a/.github/workflows/tester.yaml b/.github/workflows/tester.yaml index 14d22ce..4d57db1 100644 --- a/.github/workflows/tester.yaml +++ b/.github/workflows/tester.yaml @@ -25,5 +25,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + - name: Validate DOSDP Schema + run: python ./src/schema/schema_validator.py - name: Run Test run: PYTHONPATH=./src:$PYTHONPATH python -m unittest discover -s src -p '*_test.py' diff --git a/src/schema/dosdp_schema.yaml b/src/schema/dosdp_schema.yaml index f6a10ae..42750b7 100644 --- a/src/schema/dosdp_schema.yaml +++ b/src/schema/dosdp_schema.yaml @@ -334,7 +334,7 @@ properties: doc_type: root examples: - type: list + type: array items: { type: string } description: "A list of example terms implementing this pattern." doc_type: root diff --git a/src/schema/schema_validator.py b/src/schema/schema_validator.py new file mode 100644 index 0000000..fd8557b --- /dev/null +++ b/src/schema/schema_validator.py @@ -0,0 +1,12 @@ +import os +from ruamel.yaml import YAML +from jsonschema import Draft7Validator + +SCHEMA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../schema/dosdp_schema.yaml") + + +ryaml = YAML(typ='safe') +with open(SCHEMA_PATH) as stream: + dosdp_schema = ryaml.load(stream) + +Draft7Validator.check_schema(dosdp_schema) diff --git a/src/schema/test/generic_test/schema_test.py b/src/schema/test/generic_test/schema_test.py index ae14c3a..8598053 100644 --- a/src/schema/test/generic_test/schema_test.py +++ b/src/schema/test/generic_test/schema_test.py @@ -10,6 +10,8 @@ "../positive_test_set/patterns/acute.yaml") POSITIVE_PATTERN_2 = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../positive_test_set/patterns/multi_clause_schema.yaml") +POSITIVE_PATTERN_3 = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../positive_test_set/patterns/abnormallyHyperplasticAnatomicalEntity.yaml") NEGATIVE_PATTERN_1 = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../negative_test_set/acute_negative.yaml") @@ -38,6 +40,13 @@ def test_schema_validity1(self): # failing because def does not have a text self.assertEqual("{'vars': ['disease']} is not valid under any of the given schemas", next(es).message) + def test_array_validity(self): + self.assertTrue(pathlib.Path(SCHEMA).exists()) + + validator = Draft4Validator(self.read_yaml(SCHEMA)) + + self.assertTrue(validator.is_valid(self.read_yaml(POSITIVE_PATTERN_3))) + def read_yaml(self, yaml_path): with open(yaml_path, "r") as stream: try: diff --git a/src/schema/test/positive_test_set/patterns/abnormallyHyperplasticAnatomicalEntity.yaml b/src/schema/test/positive_test_set/patterns/abnormallyHyperplasticAnatomicalEntity.yaml new file mode 100644 index 0000000..d1940c1 --- /dev/null +++ b/src/schema/test/positive_test_set/patterns/abnormallyHyperplasticAnatomicalEntity.yaml @@ -0,0 +1,50 @@ +pattern_name: abnormallyHyperplasticAnatomicalEntity.yaml + +pattern_iri: http://purl.obolibrary.org/obo/upheno/patterns-dev/abnormallyHyperplasticAnatomicalEntity.yaml + +description: 'Hyperplastic anatomical entity, for example, thyroid gland hyperplasia. The increase in size of the anatomical entitiy is due to an increase in cell number,' + +examples: + - http://purl.obolibrary.org/obo/HP_0008249 + - http://purl.obolibrary.org/obo/MP_0000630 + - http://purl.obolibrary.org/obo/MP_0013765 + +contributors: + - https://orcid.org/0000-0001-8314-2140 + +classes: + abnormal: PATO:0000460 + anatomical_entity: UBERON:0001062 + hyperplastic: PATO:0000644 + +relations: + characteristic_of: RO:0000052 + has_modifier: RO:0002573 + has_part: BFO:0000051 + +annotationProperties: + exact_synonym: oio:hasExactSynonym + +vars: + anatomical_entity: "'anatomical_entity'" + +name: + text: "%s hyperplasia" + vars: + - anatomical_entity + +annotations: + - annotationProperty: exact_synonym + text: "hyperplastic %s" + vars: + - anatomical_entity + +def: + text: "The increased size of the %s is due to an increase in cell number (hyperplasia)." + vars: + - anatomical_entity + +equivalentTo: + text: "'has_part' some ('hyperplastic' and ('characteristic_of' some %s) and ('has_modifier' some 'abnormal'))" + vars: + - anatomical_entity From 8b15f6434a31b97f9e75e323a68908b5c8fb57f9 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Thu, 24 Feb 2022 16:44:26 +0000 Subject: [PATCH 02/10] schema examples list type changed to array --- src/schema/dosdp_schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/dosdp_schema.yaml b/src/schema/dosdp_schema.yaml index 42750b7..a738443 100644 --- a/src/schema/dosdp_schema.yaml +++ b/src/schema/dosdp_schema.yaml @@ -348,7 +348,7 @@ properties: doc_type: root tags: - type: list + type: array items: { type: string } description: > A list of strings used to tag a pattern for the purposes of arbitrary, From be99569a63f2bda64e9d46d407998062d5bbfe7f Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 09:39:31 +0000 Subject: [PATCH 03/10] version number increased --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e894e1..d1b69fb 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ # This call to setup() does all the work setup( name="dosdp", - version="0.1.7.dev1", + version="0.1.9.dev1", description="The aim of this project is to specify a simple OWL design pattern system that can easily be consumed, whatever your code base.", long_description=README, long_description_content_type="text/markdown", From 35c9a8667543a1581263c116aa6b425abbc90bf0 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 10:32:10 +0000 Subject: [PATCH 04/10] version number increased --- setup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/setup.py b/setup.py index d1b69fb..e251e7c 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,27 @@ import pathlib from setuptools import setup +READTHEDOCS = "(http://incatools.github.io/dead_simple_owl_design_patterns/" +relative_link_mapping = {"(docs/dosdp_schema.md)": READTHEDOCS + "dosdp_schema/)", + "(docs/validator.md)": READTHEDOCS + "validator/)", + "(docs/document.md)": READTHEDOCS + "document/)"} + + +def update_relative_links(readme_content): + """ + Relative links are broken in the pypi home page. So replace them with read the docs absolute links. + """ + for key, value in relative_link_mapping.items(): + readme_content = readme_content.replace(key, value) + return readme_content + + # The directory containing this file HERE = pathlib.Path(__file__).parent # The text of the README file README = (HERE / "README.md").read_text() +README = update_relative_links(README) # This call to setup() does all the work setup( From 1cf4152faffa63278ff8737aba372e9e7a8bbe5c Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 10:47:01 +0000 Subject: [PATCH 05/10] pypi relative links fixed --- .github/workflows/publish_to_pypi.yaml | 2 +- .github/workflows/publish_to_test_pypi.yaml | 39 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish_to_test_pypi.yaml diff --git a/.github/workflows/publish_to_pypi.yaml b/.github/workflows/publish_to_pypi.yaml index 4ad5f7c..3d4ca5b 100644 --- a/.github/workflows/publish_to_pypi.yaml +++ b/.github/workflows/publish_to_pypi.yaml @@ -40,4 +40,4 @@ jobs: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - twine upload --repository-url https://test.pypi.org/legacy/ dist/* \ No newline at end of file + twine upload --repository-url https://pypi.org/legacy/ dist/* \ No newline at end of file diff --git a/.github/workflows/publish_to_test_pypi.yaml b/.github/workflows/publish_to_test_pypi.yaml new file mode 100644 index 0000000..4844532 --- /dev/null +++ b/.github/workflows/publish_to_test_pypi.yaml @@ -0,0 +1,39 @@ +name: Publish Python distributions to PyPI + +# Expect this action to be triggered manually before + +on: + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build-and-publish: + name: Publish Python distributions to PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Deployment Tools + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Install Project Dependencies + run: | + pip install -r requirements.txt + - name: Auto-Generate schema documentation + run: PYTHONPATH=./src:$PYTHONPATH python -m dosdp document --schema -o ./src/schema/ + - name: Package Distribution + run: >- + python + setup.py + sdist + bdist_wheel + - name: Deploy Package + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} + run: | + twine upload --repository-url https://test.pypi.org/legacy/ dist/* \ No newline at end of file From 8fbf1a1906bbfc1c92de190ddee733dd8a6d4e82 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 10:49:08 +0000 Subject: [PATCH 06/10] pypi relative links fixed --- .github/workflows/publish_to_pypi.yaml | 2 +- .github/workflows/publish_to_test_pypi.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_to_pypi.yaml b/.github/workflows/publish_to_pypi.yaml index 3d4ca5b..1803bf2 100644 --- a/.github/workflows/publish_to_pypi.yaml +++ b/.github/workflows/publish_to_pypi.yaml @@ -1,4 +1,4 @@ -name: Publish Python distributions to PyPI +name: Publish to PyPI on: # Allows you to run this workflow manually from the Actions tab diff --git a/.github/workflows/publish_to_test_pypi.yaml b/.github/workflows/publish_to_test_pypi.yaml index 4844532..f23593f 100644 --- a/.github/workflows/publish_to_test_pypi.yaml +++ b/.github/workflows/publish_to_test_pypi.yaml @@ -1,4 +1,4 @@ -name: Publish Python distributions to PyPI +name: Publish to test PyPI # Expect this action to be triggered manually before From fa53f9053356696f1f99751830fd3e3a3ea18434 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 10:56:47 +0000 Subject: [PATCH 07/10] pypi release git action updated --- .github/workflows/publish_to_pypi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_to_pypi.yaml b/.github/workflows/publish_to_pypi.yaml index 1803bf2..17bc472 100644 --- a/.github/workflows/publish_to_pypi.yaml +++ b/.github/workflows/publish_to_pypi.yaml @@ -40,4 +40,4 @@ jobs: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - twine upload --repository-url https://pypi.org/legacy/ dist/* \ No newline at end of file + twine upload dist/* \ No newline at end of file From 2371372713dbe661ba8fcfd940f8772c1f6c1cf2 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 15:09:30 +0000 Subject: [PATCH 08/10] Validator does not check annotationProperty bindings #82 --- src/dosdp/validator.py | 26 ++++++++++ .../test/generic_test/validator_test.py | 5 ++ .../undeclared_annotation_prop.yaml | 47 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 src/schema/test/negative_test_set/undeclared_annotation_prop.yaml diff --git a/src/dosdp/validator.py b/src/dosdp/validator.py index ac8e8c0..3d63fce 100755 --- a/src/dosdp/validator.py +++ b/src/dosdp/validator.py @@ -200,6 +200,31 @@ def test_multi_clause_multi_list(pattern): return stat +def test_annotation_properties(pattern): + """ + Structurally tests whether an annotation property is declared before use. + Args: + pattern: schema in yaml format to validate + + Returns: True if all used annotation properties are declared, False otherwise. + """ + declared_annotations = set() + if 'annotationProperties' in pattern.keys(): declared_annotations.update(set(pattern['annotationProperties'].keys())) + expr = parse('annotations.[*].annotationProperty') + used_annotations = [match for match in expr.find(pattern)] + + print(declared_annotations) + stat = True + if used_annotations: + for annotation_prop in used_annotations: + val = annotation_prop.value + print(val) + if val not in declared_annotations: + warnings.warn("Annotation property '%s' didn't declared before use." % val) + stat = False + return stat + + def format_warning(message, category, filename, lineno, line=None): return '%s:%s: %s:%s\n' % (filename, lineno, category.__name__, message) @@ -249,6 +274,7 @@ def validate(pattern): if not test_clause_nesting(pattern): stat = False if not test_axiom_separator(pattern): stat = False if not test_multi_clause_multi_list(pattern): stat = False + if not test_annotation_properties(pattern): stat = False except YAMLError as exc: stat = False logging.error('Failed to load pattern file: ' + pattern_doc) diff --git a/src/schema/test/generic_test/validator_test.py b/src/schema/test/generic_test/validator_test.py index 037b682..80c990d 100644 --- a/src/schema/test/generic_test/validator_test.py +++ b/src/schema/test/generic_test/validator_test.py @@ -28,6 +28,8 @@ "../negative_test_set/multi_clause_with_multi_list.yaml") NEGATIVE_PATTERN_MULTI_CLAUSE_MULTI_LIST2 = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../negative_test_set/multi_clause_with_multi_list2.yaml") +NEGATIVE_PATTERN_UNDECLARED_ANNOT_PROP = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../negative_test_set/undeclared_annotation_prop.yaml") class ValidatorTest(unittest.TestCase): @@ -60,3 +62,6 @@ def test_axiom_separator(self): def test_single_list_per_multi_clause(self): self.assertFalse(validate(NEGATIVE_PATTERN_MULTI_CLAUSE_MULTI_LIST)) self.assertFalse(validate(NEGATIVE_PATTERN_MULTI_CLAUSE_MULTI_LIST2)) + + def test_undeclared_annotation_prop(self): + self.assertFalse(validate(NEGATIVE_PATTERN_UNDECLARED_ANNOT_PROP)) diff --git a/src/schema/test/negative_test_set/undeclared_annotation_prop.yaml b/src/schema/test/negative_test_set/undeclared_annotation_prop.yaml new file mode 100644 index 0000000..fbb404b --- /dev/null +++ b/src/schema/test/negative_test_set/undeclared_annotation_prop.yaml @@ -0,0 +1,47 @@ +pattern_name: acute + +pattern_iri: http://purl.obolibrary.org/obo/mondo/patterns/acute.yaml + +description: 'This pattern is applied to diseases that are described as having an acute onset, i.e. the sudden appearance of disease manifestations over a short period of time. + +Examples: [acute bronchiolitis](http://purl.obolibrary.org/obo/MONDO_0020680), + [acute liver failure](http://purl.obolibrary.org/obo/MONDO_0019542)' + +contributors: +- https://orcid.org/0000-0002-6601-2165 +- https://orcid.org/0000-0001-5208-3432 + +classes: + acute: PATO:0000389 + disease: MONDO:0000001 + +relations: + has modifier: RO:0002573 + +annotationProperties: +# exact_synonym: oio:hasExactSynonym + related_synonym: oio:hasRelatedSynonym + +vars: + disease: '''disease''' + +name: + text: acute %s + vars: + - disease + +annotations: +- annotationProperty: exact_synonym + text: '%s, acute' + vars: + - disease + +def: + text: Acute form of %s. + vars: + - disease + +equivalentTo: + text: '%s and ''has modifier'' some ''acute''' + vars: + - disease \ No newline at end of file From c7aee9e5a9dde3f6e4f8e709c2bb11295c360e1d Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 15:10:52 +0000 Subject: [PATCH 09/10] Validator does not check annotationProperty bindings #82 --- src/dosdp/validator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/dosdp/validator.py b/src/dosdp/validator.py index 3d63fce..452fd96 100755 --- a/src/dosdp/validator.py +++ b/src/dosdp/validator.py @@ -213,12 +213,10 @@ def test_annotation_properties(pattern): expr = parse('annotations.[*].annotationProperty') used_annotations = [match for match in expr.find(pattern)] - print(declared_annotations) stat = True if used_annotations: for annotation_prop in used_annotations: val = annotation_prop.value - print(val) if val not in declared_annotations: warnings.warn("Annotation property '%s' didn't declared before use." % val) stat = False From aac531847e6bfd52c5335ff1848400cd4da2948c Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 25 Feb 2022 15:32:31 +0000 Subject: [PATCH 10/10] error messages simplified #81 --- src/dosdp/validator.py | 5 +- .../test/generic_test/validator_test.py | 5 ++ .../not_schema_compliant.yaml | 55 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 src/schema/test/negative_test_set/not_schema_compliant.yaml diff --git a/src/dosdp/validator.py b/src/dosdp/validator.py index 452fd96..249e093 100755 --- a/src/dosdp/validator.py +++ b/src/dosdp/validator.py @@ -27,7 +27,7 @@ def test_jschema(validator, pattern): if not validator.is_valid(pattern): es = validator.iter_errors(pattern) for e in es: - warnings.warn(" => ".join([str(e.schema_path), str(e.message), str(e.context)])) + warnings.warn(str(e.message)) is_valid = False return is_valid @@ -262,7 +262,8 @@ def validate(pattern): stat = True for pattern_doc in pattern_docs: - logging.info("Checking %s" % pattern_doc) + if len(pattern_docs) > 1: + logging.info("Checking %s" % pattern_doc) with open(pattern_doc, "r") as stream: try: pattern = ryaml.load(stream) diff --git a/src/schema/test/generic_test/validator_test.py b/src/schema/test/generic_test/validator_test.py index 80c990d..33572df 100644 --- a/src/schema/test/generic_test/validator_test.py +++ b/src/schema/test/generic_test/validator_test.py @@ -30,6 +30,8 @@ "../negative_test_set/multi_clause_with_multi_list2.yaml") NEGATIVE_PATTERN_UNDECLARED_ANNOT_PROP = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../negative_test_set/undeclared_annotation_prop.yaml") +NEGATIVE_PATTERN_SCHEMA = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../negative_test_set/not_schema_compliant.yaml") class ValidatorTest(unittest.TestCase): @@ -65,3 +67,6 @@ def test_single_list_per_multi_clause(self): def test_undeclared_annotation_prop(self): self.assertFalse(validate(NEGATIVE_PATTERN_UNDECLARED_ANNOT_PROP)) + + def test_schema_validation(self): + self.assertFalse(validate(NEGATIVE_PATTERN_SCHEMA)) diff --git a/src/schema/test/negative_test_set/not_schema_compliant.yaml b/src/schema/test/negative_test_set/not_schema_compliant.yaml new file mode 100644 index 0000000..f3f69b8 --- /dev/null +++ b/src/schema/test/negative_test_set/not_schema_compliant.yaml @@ -0,0 +1,55 @@ +pattern_name: acute + +pattern_iri: http://purl.obolibrary.org/obo/mondo/patterns/acute.yaml + +description: 'This pattern is applied to diseases that are described as having an acute onset, i.e. the sudden appearance of disease manifestations over a short period of time. + +Examples: [acute bronchiolitis](http://purl.obolibrary.org/obo/MONDO_0020680), + [acute liver failure](http://purl.obolibrary.org/obo/MONDO_0019542)' + +contributors: +- https://orcid.org/0000-0002-6601-2165 +- https://orcid.org/0000-0001-5208-3432 + +classes: + acute: PATO:0000389 + disease: MONDO:0000001 + +relations: + has modifier: RO:0002573 + +annotationProperties: + exact_synonym: oio:hasExactSynonym + related_synonym: oio:hasRelatedSynonym + +vars: + disease: '''disease''' + +name: + text: acute %s + vars: + - disease + +# axioms is not valid +axioms: +- annotationProperty: exact_synonym + text: '%s, acute' + vars: + - disease + +annotations: +# property is not valid +- property: exact_synonym + text: '%s, acute' + vars: + - disease + +def: + text: Acute form of %s. + vars: + - disease + +equivalentTo: + text: '%s and ''has modifier'' some ''acute''' + vars: + - disease \ No newline at end of file