Refactor schema version (#1116)

* refactor: add skipvalidation to schema_version

* tests: fix tests, lint

* tests: fix broken tests

* tests: better test for schema_version

src/aind_data_schema/core/acquisition.py

@@ -4,7 +4,7 @@
 from typing import List, Literal, Optional, Union
 
 from aind_data_schema_models.process_names import ProcessName
-from pydantic import Field, field_validator
+from pydantic import Field, SkipValidation, field_validator
 
 from aind_data_schema.base import AindCoreModel, AindModel, AwareDatetimeWithDefault
 from aind_data_schema.components.coordinates import AnatomicalDirection, AxisName, ImageAxis
@@ -45,7 +45,7 @@ class Acquisition(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/acquisition.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.1"] = Field(default="1.0.1")
+    schema_version: SkipValidation[Literal["1.0.1"]] = Field(default="1.0.1")
     protocol_id: List[str] = Field(default=[], title="Protocol ID", description="DOI for protocols.io")
     experimenter_full_name: List[str] = Field(
         ...,

src/aind_data_schema/core/data_description.py

@@ -15,7 +15,7 @@
 from aind_data_schema_models.organizations import Organization
 from aind_data_schema_models.pid_names import PIDName
 from aind_data_schema_models.platforms import Platform
-from pydantic import Field, model_validator
+from pydantic import Field, SkipValidation, model_validator
 
 from aind_data_schema.base import AindCoreModel, AindModel, AwareDatetimeWithDefault
 
@@ -40,7 +40,7 @@ class DataDescription(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/data_description.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.1"] = Field(default="1.0.1")
+    schema_version: SkipValidation[Literal["1.0.1"]] = Field(default="1.0.1")
     license: Literal["CC-BY-4.0"] = Field("CC-BY-4.0", title="License")
 
     platform: Platform.ONE_OF = Field(

src/aind_data_schema/core/instrument.py

@@ -4,7 +4,7 @@
 from typing import List, Literal, Optional
 
 from aind_data_schema_models.organizations import Organization
-from pydantic import Field, ValidationInfo, field_validator
+from pydantic import Field, SkipValidation, ValidationInfo, field_validator
 
 from aind_data_schema.base import AindCoreModel, AindModel
 from aind_data_schema.components.devices import (
@@ -35,7 +35,7 @@ class Instrument(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/instrument.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.1"] = Field(default="1.0.1")
+    schema_version: SkipValidation[Literal["1.0.1"]] = Field(default="1.0.1")
 
     instrument_id: Optional[str] = Field(
         default=None,

src/aind_data_schema/core/metadata.py

@@ -8,7 +8,15 @@
 
 from aind_data_schema_models.modalities import ExpectedFiles, FileRequirement
 from aind_data_schema_models.platforms import Platform
-from pydantic import Field, PrivateAttr, ValidationError, ValidationInfo, field_validator, model_validator
+from pydantic import (
+    Field,
+    PrivateAttr,
+    SkipValidation,
+    ValidationError,
+    ValidationInfo,
+    field_validator,
+    model_validator,
+)
 
 from aind_data_schema.base import AindCoreModel
 from aind_data_schema.core.acquisition import Acquisition
@@ -61,7 +69,7 @@ class Metadata(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/metadata.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.2"] = Field(default="1.0.2")
+    schema_version: SkipValidation[Literal["1.0.2"]] = Field(default="1.0.2")
     id: UUID = Field(
         default_factory=uuid4,
         alias="_id",

src/aind_data_schema/core/procedures.py

@@ -20,7 +20,7 @@
     VolumeUnit,
     create_unit_with_value,
 )
-from pydantic import Field, field_serializer, field_validator, model_validator
+from pydantic import Field, SkipValidation, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import ValidationInfo
 from typing_extensions import Annotated
 
@@ -649,7 +649,7 @@ class Procedures(AindCoreModel):
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/procedures.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
 
-    schema_version: Literal["1.1.1"] = Field(default="1.1.1")
+    schema_version: SkipValidation[Literal["1.1.1"]] = Field(default="1.1.1")
     subject_id: str = Field(
         ...,
         description="Unique identifier for the subject. If this is not a Allen LAS ID, indicate this in the Notes.",

src/aind_data_schema/core/processing.py

@@ -5,7 +5,7 @@
 
 from aind_data_schema_models.process_names import ProcessName
 from aind_data_schema_models.units import MemoryUnit, UnitlessUnit
-from pydantic import Field, ValidationInfo, field_validator, model_validator
+from pydantic import Field, SkipValidation, ValidationInfo, field_validator, model_validator
 
 from aind_data_schema.base import AindCoreModel, AindGeneric, AindGenericType, AindModel, AwareDatetimeWithDefault
 from aind_data_schema.components.tile import Tile
@@ -124,7 +124,7 @@ class Processing(AindCoreModel):
 
     _DESCRIBED_BY_URL: str = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/processing.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.1.1"] = Field(default="1.1.1")
+    schema_version: SkipValidation[Literal["1.1.1"]] = Field(default="1.1.1")
 
     processing_pipeline: PipelineProcess = Field(
         ..., description="Pipeline used to process data", title="Processing Pipeline"

src/aind_data_schema/core/quality_control.py

@@ -4,7 +4,7 @@
 from typing import Any, List, Literal, Optional
 
 from aind_data_schema_models.modalities import Modality
-from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic import BaseModel, Field, SkipValidation, field_validator, model_validator
 
 from aind_data_schema.base import AindCoreModel, AindModel, AwareDatetimeWithDefault
 
@@ -161,7 +161,7 @@ class QualityControl(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/quality_control.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.1.1"] = Field(default="1.1.1")
+    schema_version: SkipValidation[Literal["1.1.1"]] = Field(default="1.1.1")
     evaluations: List[QCEvaluation] = Field(..., title="Evaluations")
     notes: Optional[str] = Field(default=None, title="Notes")
 

src/aind_data_schema/core/rig.py

@@ -4,7 +4,7 @@
 from typing import List, Literal, Optional, Set, Union
 
 from aind_data_schema_models.modalities import Modality
-from pydantic import Field, ValidationInfo, field_serializer, field_validator, model_validator
+from pydantic import Field, SkipValidation, ValidationInfo, field_serializer, field_validator, model_validator
 from typing_extensions import Annotated
 
 from aind_data_schema.base import AindCoreModel
@@ -51,7 +51,7 @@ class Rig(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/rig.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.1"] = Field(default="1.0.1")
+    schema_version: SkipValidation[Literal["1.0.1"]] = Field(default="1.0.1")
     rig_id: str = Field(
         ...,
         description="Unique rig identifier, name convention: <room>-<apparatus name>-<date modified YYYYMMDD>",

src/aind_data_schema/core/session.py

@@ -17,7 +17,7 @@
     TimeUnit,
     VolumeUnit,
 )
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, SkipValidation, field_validator, model_validator
 from pydantic_core.core_schema import ValidationInfo
 from typing_extensions import Annotated
 
@@ -534,7 +534,7 @@ class Session(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/session.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.1"] = Field(default="1.0.1")
+    schema_version: SkipValidation[Literal["1.0.1"]] = Field(default="1.0.1")
     protocol_id: List[str] = Field(default=[], title="Protocol ID", description="DOI for protocols.io")
     experimenter_full_name: List[str] = Field(
         ...,

src/aind_data_schema/core/subject.py

@@ -8,7 +8,7 @@
 from aind_data_schema_models.organizations import Organization
 from aind_data_schema_models.pid_names import PIDName
 from aind_data_schema_models.species import Species
-from pydantic import Field, field_validator
+from pydantic import Field, SkipValidation, field_validator
 from pydantic_core.core_schema import ValidationInfo
 
 from aind_data_schema.base import AindCoreModel, AindModel
@@ -89,7 +89,7 @@ class Subject(AindCoreModel):
 
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/subject.py"
     describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
-    schema_version: Literal["1.0.0"] = Field(default="1.0.0")
+    schema_version: SkipValidation[Literal["1.0.0"]] = Field(default="1.0.0")
     subject_id: str = Field(
         ...,
         description="Unique identifier for the subject. If this is not a Allen LAS ID, indicate this in the Notes.",

src/aind_data_schema/utils/schema_version_bump.py

@@ -101,8 +101,8 @@ def _get_updated_file(python_file_path: str, new_ver: str) -> list:
         with open(python_file_path, "rb") as f:
             file_lines = f.readlines()
         for line in file_lines:
-            if "schema_version: Literal[" in str(line):
-                new_line_str = f'    schema_version: Literal["{new_ver}"] = Field("{new_ver}")\n'
+            if "schema_version: SkipValidation[Literal[" in str(line):
+                new_line_str = f'    schema_version: SkipValidation[Literal["{new_ver}"]] = Field("{new_ver}")\n'
                 new_line = new_line_str.encode()
             else:
                 new_line = line

tests/test_bump_schema_versions.py

@@ -77,18 +77,18 @@ def test_update_files(self, mock_write: MagicMock):
         handler._update_files({Subject: new_subject_version, Session: new_session_version})
 
         expected_line_change0 = (
-            f'    schema_version: Literal["{new_subject_version}"] = Field("{new_subject_version}")\n'
+            f'schema_version: SkipValidation[Literal["{new_subject_version}"]] = Field("{new_subject_version}")'
         )
         expected_line_change1 = (
-            f'    schema_version: Literal["{new_session_version}"] = Field("{new_session_version}")\n'
+            f'schema_version: SkipValidation[Literal["{new_session_version}"]] = Field("{new_session_version}")'
         )
 
         mock_write_args0 = mock_write.mock_calls[0].args
         mock_write_args1 = mock_write.mock_calls[1].args
+        self.assertTrue(expected_line_change0 in str(mock_write_args0[0]))
         self.assertTrue("subject.py" in str(mock_write_args0[1]))
-        self.assertTrue(expected_line_change0.encode() in mock_write_args0[0])
+        self.assertTrue(expected_line_change1 in str(mock_write_args1[0]))
         self.assertTrue("session.py" in str(mock_write_args1[1]))
-        self.assertTrue(expected_line_change1.encode() in mock_write_args1[0])
 
     @patch("aind_data_schema.utils.schema_version_bump.SchemaVersionHandler._get_list_of_models_that_changed")
     @patch("aind_data_schema.utils.schema_version_bump.SchemaVersionHandler._update_files")

tests/test_metadata.py

@@ -326,6 +326,24 @@ def test_validate_rig_session_compatibility(self):
             str(context.exception),
         )
 
+    def test_validate_old_schema_version(self):
+        """Tests that old schema versions are ignored during validation
+        """
+        m = Metadata.model_construct(
+            name="name",
+            location="location",
+            id="1",
+        )
+
+        m_dict = m.model_dump()
+
+        m_dict["schema_version"] = "0.0.0"
+        m_dict.pop("id")
+
+        m2 = Metadata(**m_dict)
+
+        self.assertIsNotNone(m2)
+
 
 if __name__ == "__main__":
     unittest.main()