Feat 253 upgrade schema (#255)

* fix: nsb2023, nsb2019, las2020, labtracks, smartsheet * fix: comments out OptionalEnumMeta * feat: adds upper bound to pydantic * feat: updates aind-metadata-mapper to v0.17.0, which install aind-data-schema v1.0.0 --------- Co-authored-by: jtyoung84 <[email protected]>
AllenNeuralDynamics · Sep 7, 2024 · c8e6b5b · c8e6b5b
1 parent a4b39bd
commit c8e6b5b
Show file tree

Hide file tree

Showing 39 changed files with 127 additions and 115 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -17,9 +17,9 @@ readme = "README.md"
 dynamic = ["version"]
 
 dependencies = [
-    'aind-metadata-mapper==0.6.1',
     'pydantic-settings>=2.0',
-    'pydantic<=2.6.4'
+    'pydantic>=2.7,<2.9',
+    'aind-metadata-mapper==0.17.0'
 ]
 
 [project.optional-dependencies]

diff --git a/src/aind_metadata_service/labtracks/client.py b/src/aind_metadata_service/labtracks/client.py
@@ -20,7 +20,7 @@
     Species,
     Subject,
 )
-from aind_data_schema.models.organizations import Organization
+from aind_data_schema_models.organizations import Organization
 from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings
 

diff --git a/src/aind_metadata_service/sharepoint/las2020/models.py b/src/aind_metadata_service/sharepoint/las2020/models.py
@@ -11,12 +11,14 @@
 class OptionalEnumMeta(EnumMeta):
     """Return None if constructing from a value not in members."""
 
-    def __call__(cls, value, *args, **kw):
-        """Return none if value not in members. Else return enum."""
-        if value in [m.value for m in cls.__members__.values()]:
-            return super().__call__(value, *args, **kw)
-        else:
-            return None
+    # TODO: No longer works bc pydantic 2.7 stricter validation.
+    #  Handle when enum value not in members.
+    # def __call__(cls, value, *args, **kw):
+    #     """Return none if value not in members. Else return enum."""
+    #     if value in [m.value for m in cls.__members__.values()]:
+    #         return super().__call__(value, *args, **kw)
+    #     else:
+    #         return None
 
 
 class Bctype(Enum, metaclass=OptionalEnumMeta):

diff --git a/src/aind_metadata_service/sharepoint/nsb2019/mapping.py b/src/aind_metadata_service/sharepoint/nsb2019/mapping.py
@@ -23,7 +23,7 @@
     ViralMaterial,
 )
 from aind_data_schema.core.subject import Sex
-from aind_data_schema.models.devices import FiberProbe
+from aind_data_schema.components.devices import FiberProbe
 
 from aind_metadata_service.sharepoint.nsb2019.models import NSBList
 from aind_metadata_service.sharepoint.nsb2019.models import (

diff --git a/src/aind_metadata_service/sharepoint/nsb2019/models.py b/src/aind_metadata_service/sharepoint/nsb2019/models.py
@@ -11,12 +11,14 @@
 class OptionalEnumMeta(EnumMeta):
     """Return None if constructing from a value not in members."""
 
-    def __call__(cls, value, *args, **kw):
-        """Return none if value not in members. Else return enum."""
-        if value in [m.value for m in cls.__members__.values()]:
-            return super().__call__(value, *args, **kw)
-        else:
-            return None
+    # TODO: No longer works bc pydantic 2.7 stricter validation.
+    #  Handle when enum value not in members.
+    # def __call__(cls, value, *args, **kw):
+    #     """Return none if value not in members. Else return enum."""
+    #     if value in [m.value for m in cls.__members__.values()]:
+    #         return super().__call__(value, *args, **kw)
+    #     else:
+    #         return None
 
 
 class After1StInjection(Enum, metaclass=OptionalEnumMeta):
@@ -349,6 +351,9 @@ class ImplantIdCoverslipType(Enum, metaclass=OptionalEnumMeta):
         "5mm stacked coverslip (with silicone coating for Neuropixel)"
     )
     SELECT = "Select..."
+    # note: added removed options
+    TS_04 = "TS-04"
+    TS_05 = "TS-05"
 
 
 class Inj1Angle(Enum, metaclass=OptionalEnumMeta):

diff --git a/src/aind_metadata_service/sharepoint/nsb2023/mapping.py b/src/aind_metadata_service/sharepoint/nsb2023/mapping.py
@@ -24,9 +24,12 @@
     ViralMaterial,
 )
 from aind_data_schema.core.subject import Sex
-from aind_data_schema.models.devices import FiberProbe
+from aind_data_schema.components.devices import FiberProbe
 
-from aind_metadata_service.sharepoint.nsb2023.models import NSBList
+from aind_metadata_service.sharepoint.nsb2023.models import (
+    NSBList,
+    WindowPlacment,
+)
 from aind_metadata_service.sharepoint.nsb2023.models import (
     Procedure as NSBProcedure,
 )

diff --git a/src/aind_metadata_service/sharepoint/nsb2023/models.py b/src/aind_metadata_service/sharepoint/nsb2023/models.py
@@ -11,12 +11,14 @@
 class OptionalEnumMeta(EnumMeta):
     """Return None if constructing from a value not in members."""
 
-    def __call__(cls, value, *args, **kw):
-        """Return none if value not in members. Else return enum."""
-        if value in [m.value for m in cls.__members__.values()]:
-            return super().__call__(value, *args, **kw)
-        else:
-            return None
+    # TODO: No longer works bc pydantic 2.7 stricter validation.
+    #  Handle when enum value not in members.
+    # def __call__(cls, value, *args, **kw):
+    #     """Return none if value not in members. Else return enum."""
+    #     if value in [m.value for m in cls.__members__.values()]:
+    #         return super().__call__(value, *args, **kw)
+    #     else:
+    #         return None
 
 
 class Behavior(Enum, metaclass=OptionalEnumMeta):
@@ -952,6 +954,9 @@ class Lightcycle(Enum, metaclass=OptionalEnumMeta):
 
     REVERSE_LIGHT_CYCLE_9PM_T = "Reverse Light Cycle (9pm to 9am light)"
     STANDARD_LIGHT_CYCLE_6AM = "Standard Light Cycle (6am to 8pm light)"
+    # ADDED FIELDS FROM OLDER ENTRIES
+    REVERSE_9PM_TO_9AM = "Reverse (9pm to 9am)"
+    STANDARD_6AM_TO_8PM = "Standard (6am to 8pm)"
 
 
 class LimsRequired(Enum, metaclass=OptionalEnumMeta):
@@ -1445,6 +1450,7 @@ class Projectid(Enum, metaclass=OptionalEnumMeta):
     N_1210101510_MSP_FALCONWO = "121-01-015-10 MSP Falconwood"
     N_1210199910_MSP_CROSS_PR = "121-01-999-10 MSP Cross Program 10YP"
     N_1210199910_MSP_CROS_001 = "121-01-999-10 MSP Cross Program  Scientific"
+    N_1210101210_LEARNING_MFISH = "121-01-012-10 Learning mFISH"
 
 
 class Sex(Enum, metaclass=OptionalEnumMeta):

diff --git a/src/aind_metadata_service/smartsheet/funding/mapping.py b/src/aind_metadata_service/smartsheet/funding/mapping.py
@@ -5,7 +5,7 @@
 from typing import List, Optional, Union
 
 from aind_data_schema.core.data_description import Funding
-from aind_data_schema.models.organizations import Organization
+from aind_data_schema_models.organizations import Organization
 from pydantic import ValidationError
 from starlette.responses import JSONResponse
 
@@ -38,10 +38,7 @@ def _parse_institution(input_name: str) -> Union[Organization, str]:
         if Organization().name_map.get(input_name) is not None:
             return Organization().name_map.get(input_name)
         else:
-            try:
-                return Organization.from_abbreviation(input_name)
-            except KeyError:
-                return input_name
+            return input_name
 
     def _map_row_to_funding(
         self, row: SheetRow, input_project_name: str

diff --git a/tests/labtracks/test_response_handler.py b/tests/labtracks/test_response_handler.py
@@ -18,7 +18,7 @@
     Species,
     Subject,
 )
-from aind_data_schema.models.organizations import Organization
+from aind_data_schema_models.organizations import Organization
 
 from aind_metadata_service.labtracks.client import (
     LabTracksBgStrain,

diff --git a/tests/resources/sharepoint/las2020/raw/list_item1.json b/tests/resources/sharepoint/las2020/raw/list_item1.json
@@ -9,7 +9,7 @@
   "AuthorId": 5358,
   "RequestStatus2": "New",
   "Protocol": "2212 - Investigating Brain States",
-  "ProjectID": "Some_Project_ID",
+  "ProjectID": "AAV production 102-88-004-10",
   "AccommodationComment": "This animal will be used for an acute ephys experiment on Friday morning.",
   "ACSFID1": null,
   "ACSFID2": null,

diff --git a/tests/resources/sharepoint/las2020/raw/list_item2.json b/tests/resources/sharepoint/las2020/raw/list_item2.json
@@ -9,7 +9,7 @@
   "AuthorId": 5358,
   "RequestStatus2": "New",
   "Protocol": "2212 - Investigating Brain States",
-  "ProjectID": "Some_Project_ID",
+  "ProjectID": null,
   "AccommodationComment": "This animal will be used for an acute ephys experiment on Friday morning.",
   "ACSFID1": null,
   "ACSFID2": null,

diff --git a/tests/resources/sharepoint/las2020/raw/list_item3.json b/tests/resources/sharepoint/las2020/raw/list_item3.json
@@ -9,7 +9,7 @@
   "AuthorId": 5358,
   "RequestStatus2": "New",
   "Protocol": "2212 - Investigating Brain States",
-  "ProjectID": "Some_Project_ID",
+  "ProjectID": "AAV production 102-88-004-10",
   "AccommodationComment": "This animal will be used for an acute ephys experiment on Friday morning.",
   "ACSFID1": null,
   "ACSFID2": null,

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item12.json b/tests/resources/sharepoint/nsb2019/raw/list_item12.json
@@ -78,7 +78,7 @@
   "SecondInjectionWeightAfter": null,
   "SecondInjectionIsoDuration": null,
   "Inj1Round": "1st",
-  "Inj2Round": "N/A",
+  "Inj2Round": "NA",
   "HPIsoLevel": "Select...",
   "Round1InjIsolevel": "Select...",
   "Round2InjIsolevel": "Select...",

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item19.json b/tests/resources/sharepoint/nsb2019/raw/list_item19.json
@@ -78,7 +78,7 @@
   "SecondInjectionWeightAfter": null,
   "SecondInjectionIsoDuration": null,
   "Inj1Round": "1st",
-  "Inj2Round": "N/A",
+  "Inj2Round": "NA",
   "HPIsoLevel": "Select...",
   "Round1InjIsolevel": "Select...",
   "Round2InjIsolevel": "Select...",

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item2.json b/tests/resources/sharepoint/nsb2019/raw/list_item2.json
@@ -140,7 +140,7 @@
   "End_x0020_of_x0020_Week": "2022-12-09T08:00:00Z",
   "Age_x0020_at_x0020_Injection": "44901.0000000000",
   "CraniotomyType": "WHC NP",
-  "ImplantIDCoverslipType": "5mm coverslip",
+  "ImplantIDCoverslipType": "5mm stacked coverslip",
   "Inj1Angle_v2": "0",
   "Inj2Angle_v2": "0",
   "FiberImplant1": false,

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item3.json b/tests/resources/sharepoint/nsb2019/raw/list_item3.json
@@ -20,7 +20,7 @@
   "Virus_x0020_M_x002f_L": "-3.3",
   "Virus_x0020_A_x002f_P": "-1.6",
   "Virus_x0020_D_x002f_V": "Anywhere",
-  "Virus_x0020_Hemisphere": "Select ...",
+  "Virus_x0020_Hemisphere": "Select...",
   "HP_x0020_M_x002f_L": "2.8",
   "HP_x0020_A_x002f_P": "1.3",
   "HP_x0020_Diameter": "3mm",
@@ -78,7 +78,7 @@
   "SecondInjectionWeightAfter": null,
   "SecondInjectionIsoDuration": null,
   "Inj1Round": "1st",
-  "Inj2Round": "N/A",
+  "Inj2Round": "NA",
   "HPIsoLevel": "Select...",
   "Round1InjIsolevel": "Select...",
   "Round2InjIsolevel": "Select...",
@@ -140,7 +140,7 @@
   "End_x0020_of_x0020_Week": "2022-12-09T08:00:00Z",
   "Age_x0020_at_x0020_Injection": "44901.0000000000",
   "CraniotomyType": "Frontal Window 3mm",
-  "ImplantIDCoverslipType": "3mm",
+  "ImplantIDCoverslipType": null,
   "Inj1Angle_v2": "0",
   "Inj2Angle_v2": "0",
   "FiberImplant1": true,

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item4.json b/tests/resources/sharepoint/nsb2019/raw/list_item4.json
@@ -20,7 +20,7 @@
   "Virus_x0020_M_x002f_L": "-3.3",
   "Virus_x0020_A_x002f_P": "-1.6",
   "Virus_x0020_D_x002f_V": "Anywhere",
-  "Virus_x0020_Hemisphere": "Select ...",
+  "Virus_x0020_Hemisphere": "Select...",
   "HP_x0020_M_x002f_L": "2.8",
   "HP_x0020_A_x002f_P": "1.3",
   "HP_x0020_Diameter": "3mm",
@@ -78,7 +78,7 @@
   "SecondInjectionWeightAfter": null,
   "SecondInjectionIsoDuration": null,
   "Inj1Round": "1st",
-  "Inj2Round": "N/A",
+  "Inj2Round": "NA",
   "HPIsoLevel": "Select...",
   "Round1InjIsolevel": "Select...",
   "Round2InjIsolevel": "Select...",
@@ -139,8 +139,8 @@
   "Start_x0020_Of_x0020_Week": "2022-12-05T08:00:00Z",
   "End_x0020_of_x0020_Week": "2022-12-09T08:00:00Z",
   "Age_x0020_at_x0020_Injection": "44901.0000000000",
-  "CraniotomyType": "Other",
-  "ImplantIDCoverslipType": "3mm",
+  "CraniotomyType": "Select...",
+  "ImplantIDCoverslipType": "Select...",
   "Inj1Angle_v2": "0",
   "Inj2Angle_v2": "0",
   "FiberImplant1": true,

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item5.json b/tests/resources/sharepoint/nsb2019/raw/list_item5.json
@@ -140,7 +140,7 @@
   "End_x0020_of_x0020_Week": "2022-12-09T08:00:00Z",
   "Age_x0020_at_x0020_Injection": "44901.0000000000",
   "CraniotomyType": "Visual Cortex 5mm",
-  "ImplantIDCoverslipType": "5mm",
+  "ImplantIDCoverslipType": "5mm stacked coverslip",
   "Inj1Angle_v2": "0",
   "Inj2Angle_v2": "0",
   "FiberImplant1": true,

diff --git a/tests/resources/sharepoint/nsb2019/raw/list_item7.json b/tests/resources/sharepoint/nsb2019/raw/list_item7.json
@@ -78,7 +78,7 @@
   "SecondInjectionWeightAfter": null,
   "SecondInjectionIsoDuration": null,
   "Inj1Round": "1st",
-  "Inj2Round": "N/A",
+  "Inj2Round": "NA",
   "HPIsoLevel": "Select...",
   "Round1InjIsolevel": "Select...",
   "Round2InjIsolevel": "Select...",
@@ -140,7 +140,7 @@
   "End_x0020_of_x0020_Week": "2022-12-09T08:00:00Z",
   "Age_x0020_at_x0020_Injection": "44901.0000000000",
   "CraniotomyType": null,
-  "ImplantIDCoverslipType": "5mm",
+  "ImplantIDCoverslipType": "5mm stacked coverslip",
   "Inj1Angle_v2": "0",
   "Inj2Angle_v2": "0",
   "FiberImplant1": false,

diff --git a/tests/resources/sharepoint/nsb2023/raw/list_item10.json b/tests/resources/sharepoint/nsb2023/raw/list_item10.json
@@ -120,9 +120,9 @@
   "ComSwelling": "None",
   "ComSinusbleed": "Mild",
   "ComDamage": "None",
-  "ComWindow": "None",
+  "ComWindow": null,
   "ComCoplanar": "None",
-  "Contusion": "Central",
+  "Contusion": "Mild",
   "IontoNumberInj1": "Select...",
   "NanojectNumberInj10": "Select...",
   "IontoNumberInj2": "Select...",
@@ -146,7 +146,7 @@
   "Headpost_x0020_Perform_x0020_Dur": null,
   "Craniotomy_x0020_Perform_x0020_D": "Initial Surgery",
   "ProjectID": "121-01-012-10 Learning mFISH",
-  "LightCycle": "Reverse (9pm to 9am)",
+  "LightCycle": "Reverse Light Cycle (9pm to 9am light)",
   "LIMSProject": "LearningmFISHDevelopment",
   "LIMSTaskflow": "MSP Learning & mFISH Development (Dox)",
   "OData__x0023__x0020_of_x0020_Burr_x002": "1",
@@ -218,7 +218,7 @@
   "Burr5_x0020_Injection_x0020_Devi": "Select...",
   "Burr6_x0020_Injection_x0020_Devi": "Select...",
   "OData__ColorTag": null,
-  "Behavior": "0",
+  "Behavior": "No",
   "Behavior_x0020_Type": "Select...",
   "Behavior_x0020_Complete": null,
   "Burr_x0020_1_x0020_Injectable_x0": "230929-12",

diff --git a/tests/resources/sharepoint/nsb2023/raw/list_item11.json b/tests/resources/sharepoint/nsb2023/raw/list_item11.json
@@ -120,9 +120,9 @@
   "ComSwelling": "None",
   "ComSinusbleed": "Mild",
   "ComDamage": "None",
-  "ComWindow": "None",
+  "ComWindow": "Select...",
   "ComCoplanar": "None",
-  "Contusion": "Central",
+  "Contusion": "Moderate",
   "IontoNumberInj1": "Select...",
   "NanojectNumberInj10": "Select...",
   "IontoNumberInj2": "Select...",
@@ -218,7 +218,7 @@
   "Burr5_x0020_Injection_x0020_Devi": "Select...",
   "Burr6_x0020_Injection_x0020_Devi": "Select...",
   "OData__ColorTag": null,
-  "Behavior": "0",
+  "Behavior": null,
   "Behavior_x0020_Type": "Select...",
   "Behavior_x0020_Complete": null,
   "Burr_x0020_1_x0020_Injectable_x0": "230929-12",

diff --git a/tests/resources/sharepoint/nsb2023/raw/list_item13.json b/tests/resources/sharepoint/nsb2023/raw/list_item13.json
@@ -41,7 +41,7 @@
   "SurgeryStatus": "Ready for Feedback",
   "LongRequestorComments": null,
   "Procedure_x0020_Slots": "Single surgical session",
-  "Procedure_x0020_Family": "Headpost Only",
+  "Procedure_x0020_Family": "Headpost only",
   "Procedure_x0020_T2": "Select...",
   "Procedure": null,
   "Headpost": "Visual Ctx",
@@ -120,9 +120,9 @@
   "ComSwelling": "None",
   "ComSinusbleed": "Mild",
   "ComDamage": "None",
-  "ComWindow": "None",
+  "ComWindow": "N/A",
   "ComCoplanar": "None",
-  "Contusion": "Central",
+  "Contusion": "N/A",
   "IontoNumberInj1": "Select...",
   "NanojectNumberInj10": "Select...",
   "IontoNumberInj2": "Select...",
@@ -140,7 +140,7 @@
   "Attachments": false,
   "GUID": "cda0b241-be4f-4a14-a7ef-75a01939036a",
   "ProjectID": "121-01-012-10 Learning mFISH",
-  "LightCycle": "Reverse (9pm to 9am)",
+  "LightCycle": "Reverse Light Cycle (9pm to 9am light)",
   "LIMSProject": "LearningmFISHDevelopment",
   "LIMSTaskflow": "MSP Learning & mFISH Development (Dox)",
   "OData__x0023__x0020_of_x0020_Burr_x002": "1",
@@ -217,7 +217,7 @@
   "Burr5_x0020_Injection_x0020_Devi": "Select...",
   "Burr6_x0020_Injection_x0020_Devi": "Select...",
   "OData__ColorTag": null,
-  "Behavior": "0",
+  "Behavior": "Yes",
   "Behavior_x0020_Type": "Select...",
   "Behavior_x0020_Complete": null,
   "Burr_x0020_1_x0020_Injectable_x0": "230929-12",