🔀 Merge pull request #131 from cnr-ibba/devel

🔖 release 0.4.10
cnr-ibba · May 27, 2024 · d6f7bf1 · d6f7bf1
2 parents fcdb3ce + 1070cb1
commit d6f7bf1
Show file tree

Hide file tree

Showing 36 changed files with 5,405 additions and 524 deletions.
diff --git a/.github/workflows/lint-workflow.yml b/.github/workflows/lint-workflow.yml
@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Cache conda
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           # Increase this value to reset cache if environment.yml has not changed
           CACHE_NUMBER: 0
@@ -32,22 +32,23 @@ jobs:
             hashFiles('environment.yml') }}
 
       - name: Setup conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
+          auto-update-conda: false
           activate-environment: SMARTER-database
           channel-priority: strict
-          environment-file: environment.yml
+          environment-file: conda-linux-64.lock
           use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
-      - name: Cache pip
-        uses: actions/cache@v3
+      - name: Cache poetry
+        uses: actions/cache@v4
         with:
           # This path is specific to Ubuntu
-          path: ~/.cache/pip
+          path: /.cache/pypoetry
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
           restore-keys: |
-            ${{ runner.os }}-pip-
+            ${{ runner.os }}-poetry-
             ${{ runner.os }}-
 
       - name: Conda info

diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml
@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Cache conda
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           # Increase this value to reset cache if environment.yml has not changed
           CACHE_NUMBER: 0
@@ -32,22 +32,23 @@ jobs:
             hashFiles('environment.yml') }}
 
       - name: Setup conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
+          auto-update-conda: false
           activate-environment: SMARTER-database
           channel-priority: strict
-          environment-file: environment.yml
+          environment-file: conda-linux-64.lock
           use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
-      - name: Cache pip
-        uses: actions/cache@v3
+      - name: Cache poetry
+        uses: actions/cache@v4
         with:
           # This path is specific to Ubuntu
-          path: ~/.cache/pip
+          path: /.cache/pypoetry
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
           restore-keys: |
-            ${{ runner.os }}-pip-
+            ${{ runner.os }}-poetry-
             ${{ runner.os }}-
 
       - name: Conda info

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -7,22 +7,24 @@ version: 2
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.9"
-    # You can also specify other tool versions:
-    # nodejs: "16"
-    # rust: "1.55"
-    # golang: "1.17"
+    python: "3.12"
+  jobs:
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry
+    post_install:
+      # Install dependencies with 'docs' dependency group
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      # VIRTUAL_ENV needs to be set manually for now.
+      # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
 
 # If using Sphinx, optionally build your docs in additional formats such as PDF
 formats: all
-
-# Optionally declare the Python requirements required to build your docs
-python:
-  install:
-  - requirements: requirements.txt
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -11,15 +11,24 @@
         "dotenv",
         "Ensembl",
         "Frizarta",
+        "geocoders",
+        "geodatasets",
+        "geopandas",
+        "geopy",
         "INITDB",
         "mapdata",
         "mongodump",
+        "mongoengine",
         "mongoexpress",
+        "mongomock",
         "mongorestore",
+        "naturalearth",
+        "Nominatim",
         "pardir",
         "pedfile",
         "plinkio",
         "probeset",
+        "pycountry",
         "SGID",
         "smarterdb",
         "SNPCHIMP",

diff --git a/HISTORY.rst b/HISTORY.rst
@@ -12,11 +12,12 @@ TODO
   in database (which will be used as reference)
 * Map affymetrix snps in *OARV3* coordinates
 * Check if ``rs_id`` is still valid or not (with EVA)
-* Manage python packages with `poetry <https://python-poetry.org/>`__
 
 0.4.10.dev0
 -----------
 
+* Fix issues with samples using reverse geocoding (`#112 <https://github.com/cnr-ibba/SMARTER-database/issues/112>`__)
+* Manage python packages with `poetry <https://python-poetry.org/>`__ (`#128 <https://github.com/cnr-ibba/SMARTER-database/issues/128>`__)
 * Add data for Guisandesa goats (`#117 <https://github.com/cnr-ibba/SMARTER-database/issues/117>`)
 * Rename ``manifacturer`` into ``manufacturer``
 * Convert genotypes from *top* to *forward* (`#111 <https://github.com/cnr-ibba/SMARTER-database/issues/111>`__)

diff --git a/Makefile b/Makefile
@@ -25,7 +25,7 @@ endif
 
 ## Install Python Dependencies
 requirements: test_environment
-	$(PYTHON_INTERPRETER) -m pip install -r requirements.txt
+	poetry install
 
 ## Initialize database by loading stuff
 initialize: requirements
@@ -92,6 +92,7 @@ data: requirements
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Texel --code TEX --alias TEXEL_UY --dataset TEXEL_INIA_UY.zip
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Frizarta --code FRZ --alias 0 --dataset Frizarta54samples_ped_map_files.zip
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Merino --code MER --alias MERINO_UY --dataset MERINO_INIA_UY.zip
+	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Merino --code MER --alias MER --dataset MERINO_INIA_UY.zip
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Corriedale --code CRR --alias CORRIEDALE_UY --dataset CORRIEDALE_INIA_UY.zip
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name Creole --code CRL --alias CRL --dataset CREOLE_INIA_UY.zip
 	$(PYTHON_INTERPRETER) src/data/add_breed.py --species_class sheep --name "Mérinos d'Arles" --code ARL --alias MER --dataset="High density genotypes of French Sheep populations.zip"
@@ -147,7 +148,7 @@ data: requirements
 
 	## load breeds into database relying on dataset
 	$(PYTHON_INTERPRETER) src/data/import_breeds.py --species_class Sheep --src_dataset="High density genotypes of French Sheep populations.zip" \
-		--datafile Populations_infos_fix.xlsx --code_column Code --breed_column "Population Name"
+		--datafile Populations_infos_fix.xlsx --code_column Code --breed_column "Population Name" --country_column Country
 	$(PYTHON_INTERPRETER) src/data/import_breeds.py --species_class Sheep --src_dataset=ovine_SNP50HapMap_data.zip \
 		--datafile ovine_SNP50HapMap_data/kijas2012_dataset_fix.xlsx --code_column code --breed_column Breed \
 		--fid_column Breed --country_column country
@@ -174,7 +175,7 @@ data: requirements
 		--datafile 41598_2017_7382_MOESM2_ESM/barbato_sheep_metadata.xlsx --code_column code --breed_column breed_x \
 		--fid_column code --country_column country_x
 	$(PYTHON_INTERPRETER) src/data/import_breeds.py --species_class Sheep --src_dataset Ciani_2020.zip \
-		--datafile 8947346/ciani_2020_metadata.xlsx --code_column code --breed_column breed \
+		--datafile 8947346/ciani_2020_metadata_fix.xlsx --code_column code --breed_column breed \
 		--fid_column fid --country_column country
 	$(PYTHON_INTERPRETER) src/data/import_breeds.py --species_class Sheep --src_dataset northwest_africa_sheep.zip \
 		--datafile northwest_africa_sheep/belabdi_2019_metadata.xlsx --code_column code --breed_column breed \
@@ -196,9 +197,9 @@ data: requirements
 	$(foreach ASSEMBLY, $(SHEEP_ASSEMBLIES), $(PYTHON_INTERPRETER) src/data/import_from_plink.py --file Frizarta54samples_ped_map_files/Frizarta54samples \
 		--dataset Frizarta54samples_ped_map_files.zip --src_coding forward --chip_name IlluminaOvineSNP50 \
 		--assembly $(ASSEMBLY) --create_samples;)
-	$(foreach ASSEMBLY, $(SHEEP_ASSEMBLIES),  $(PYTHON_INTERPRETER) src/data/import_from_plink.py --file MERINO_UY_96_21_12_17_OV54k \
-		--dataset MERINO_INIA_UY.zip --chip_name IlluminaOvineSNP50 \
-		--assembly $(ASSEMBLY) --create_samples;)
+	$(PYTHON_INTERPRETER) src/data/import_samples.py --src_dataset MERINO_INIA_UY.zip \
+		--datafile MERINO_UY_96_21_12_17_OV54k_samples.xlsx --code_column code --id_column iid \
+		--chip_name IlluminaOvineSNP50 --country_column country
 	$(foreach ASSEMBLY, $(SHEEP_ASSEMBLIES), $(PYTHON_INTERPRETER) src/data/import_from_plink.py --file CORRIEDALE_UY_60_INIA_Ovine_14sep2010 \
 		--dataset CORRIEDALE_INIA_UY.zip --chip_name IlluminaOvineSNP50 \
 		--assembly $(ASSEMBLY) --create_samples;)
@@ -326,7 +327,7 @@ data: requirements
 		--datafile 41598_2017_7382_MOESM2_ESM/barbato_sheep_metadata.xlsx --code_column code --id_column original_id \
 		--chip_name IlluminaOvineSNP50 --country_column country_x --sex_column sex
 	$(PYTHON_INTERPRETER) src/data/import_samples.py --src_dataset Ciani_2020.zip \
-		--datafile 8947346/ciani_2020_metadata.xlsx --code_column fid --id_column original_id \
+		--datafile 8947346/ciani_2020_metadata_fix.xlsx --code_column fid --id_column original_id \
 		--chip_name IlluminaOvineSNP50 --country_column country --species_column species
 	$(PYTHON_INTERPRETER) src/data/import_samples.py --src_dataset northwest_africa_sheep.zip \
 		--datafile northwest_africa_sheep/belabdi_2019_metadata.xlsx --code_column fid --id_column original_id \
@@ -336,6 +337,8 @@ data: requirements
 		--chip_name IlluminaOvineSNP50 --country_column Country
 
 	## convert genotypes without creating samples in database (SHEEP)
+	$(foreach ASSEMBLY, $(SHEEP_ASSEMBLIES),  $(PYTHON_INTERPRETER) src/data/import_from_plink.py --file MERINO_UY_96_21_12_17_OV54k \
+		--dataset MERINO_INIA_UY.zip --chip_name IlluminaOvineSNP50 --assembly $(ASSEMBLY);)
 	$(foreach ASSEMBLY, $(SHEEP_ASSEMBLIES), $(PYTHON_INTERPRETER) src/data/import_from_affymetrix.py --prefix Affymetrix_data_Plate_652_660/Affymetrix_data_Plate_652/Affymetrix_data_Plate_652 \
 		--dataset Affymetrix_data_Plate_652_660.zip --breed_code CRR --chip_name AffymetrixAxiomOviCan --assembly $(ASSEMBLY) --sample_field alias \
 		--src_version Oar_v4.0 --src_imported_from affymetrix;)
@@ -594,7 +597,7 @@ data: requirements
 		--metadata_column sample_accession --metadata_column sample_provider --metadata_column closest_city \
 		--metadata_column closest_locality --metadata_column estimated_age_months --metadata_column sampling_date
 	$(PYTHON_INTERPRETER) src/data/import_metadata.py --src_dataset Ciani_2020.zip \
-		--datafile 8947346/ciani_2020_metadata.xlsx --id_column original_id \
+		--datafile 8947346/ciani_2020_metadata_fix.xlsx --id_column original_id \
 		--latitude_column latitude --longitude_column longitude --metadata_column region \
 		--metadata_column Type
 	$(PYTHON_INTERPRETER) src/data/import_metadata.py --src_dataset northwest_africa_sheep.zip \
@@ -772,7 +775,7 @@ lint:
 create_environment:
 ifeq (True,$(HAS_CONDA))
 	@echo ">>> Detected conda, creating conda environment."
-	conda env create -f environment.yml
+	conda create --name $(PROJECT_NAME) --file conda-linux-64.lock
 	@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
 else
 	$(PYTHON_INTERPRETER) -m pip install -q virtualenv virtualenvwrapper

diff --git a/README.md b/README.md
@@ -55,14 +55,13 @@ Project Organization
     ├── LICENSE             <- Project LICENSE
     ├── Makefile            <- Makefile with commands like `make data` or `make train`
     ├── README.md           <- The top-level README for developers using this project.
-    ├── TODO.md             <- Stuff that need to be done
+    ├── conda-linux-64.lock <- Conda environment lock file for linux 64 bit
     ├── environment.yml     <- Conda environment file
+    ├── poetry.lock         <- Poetry lock file
+    ├── pyproject.toml      <- Poetry project file
     ├── pytest.ini          <- Configuration file for `pytest` testing environment
-    ├── requirements.txt    <- The requirements file for reproducing the analysis environment, e.g.
-    │                          generated with `pip freeze > requirements.txt`
     ├── setup.py            <- makes project pip installable (pip install -e .) so src can be imported
-    ├── test_environment.py <- Helper script to test if environment is properly set
-    └── tox.ini             <- Tox file with settings for running tox; see tox.readthedocs.io
+    └── test_environment.py <- Helper script to test if environment is properly set
 
 --------