From 205074048ce6b0df8d3cd57b7a2c798d5819f67d Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 4 Oct 2024 07:51:31 -0700 Subject: [PATCH 01/23] feat: replace processing _unit validator with generic _unit validator --- src/aind_data_schema/base.py | 21 +++++++++++++++++++++ src/aind_data_schema/core/processing.py | 9 --------- tests/test_processing.py | 4 ++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index c04cd3c74..2a4998acb 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -16,6 +16,7 @@ create_model, ) from pydantic.functional_validators import WrapValidator +from pydantic import model_validator from typing_extensions import Annotated @@ -48,6 +49,26 @@ class AindModel(BaseModel, Generic[AindGenericType]): model_config = ConfigDict(extra="forbid", use_enum_values=True) + @model_validator(mode="after") + def unit_validator(cls, values): + """Ensure that all fields with the suffix _unit have a matching value + and if the value is set (!= None) then the unit is also set + """ + value_fields = {} + for (unit_field, unit_value) in values: + if "_unit" in unit_field: + field = unit_field.rsplit("_unit", 1)[0] + value_fields[field] = (unit_field, unit_value) + + for (field, value) in values: + if field in value_fields.keys(): + # matching value for a unit that we have + if value and not value_fields[field][1]: + # validation error + raise ValueError(f"{field}_unit is required if {field} is provided.") + + return values + class AindCoreModel(AindModel): """Generic base class to hold common fields/validators/etc for all basic AIND schema""" diff --git a/src/aind_data_schema/core/processing.py b/src/aind_data_schema/core/processing.py index e516fa7c6..d8b73c924 100644 --- a/src/aind_data_schema/core/processing.py +++ b/src/aind_data_schema/core/processing.py @@ -43,15 +43,6 @@ class ResourceUsage(AindModel): ram_usage: Optional[List[ResourceTimestamped]] = Field(default=None, title="RAM usage") usage_unit: str = Field(default=UnitlessUnit.PERCENT, title="Usage unit") - @model_validator(mode="after") - def check_value_and_unit(cls, values): - """Ensure that all valued fields have units""" - if values.system_memory and not values.system_memory_unit: - raise ValueError("System memory unit is required if system memory is provided.") - if values.ram and not values.ram_unit: - raise ValueError("RAM unit is required if RAM is provided.") - return values - class DataProcess(AindModel): """Description of a single processing step""" diff --git a/tests/test_processing.py b/tests/test_processing.py index 81d7d9d8e..0f5952c8d 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -94,7 +94,7 @@ def test_resource_usage_unit_validators(self): ram=1, ) - expected_exception = "RAM unit is required if RAM is provided" + expected_exception = "ram_unit is required if ram is provided" self.assertTrue(expected_exception in repr(e.exception)) @@ -116,7 +116,7 @@ def test_resource_usage_unit_validators(self): system_memory=1, ) - expected_exception = "System memory unit is required if system memory is provided" + expected_exception = "system_memory_unit is required if system_memory is provided" self.assertTrue(expected_exception in repr(e.exception)) From 82f30e3053cf68843cee2c3bc4ce87e9b4420f1f Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 21 Oct 2024 20:06:01 -0700 Subject: [PATCH 02/23] feat: improve code clarity and add multi-variable --- src/aind_data_schema/base.py | 43 ++++++++++++++++++++----------- tests/test_base.py | 49 ++++++++++++++++++++++++++++++++++-- tests/test_processing.py | 4 +-- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index 2a4998acb..9f606e8e2 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -51,21 +51,36 @@ class AindModel(BaseModel, Generic[AindGenericType]): @model_validator(mode="after") def unit_validator(cls, values): - """Ensure that all fields with the suffix _unit have a matching value - and if the value is set (!= None) then the unit is also set + """Ensure that all fields matching the pattern variable_unit are set if + they have a matching variable that is set (!= None) + + This also checks the multi-variable condition, i.e. variable_unit is set + if any of variable_* are set """ - value_fields = {} - for (unit_field, unit_value) in values: - if "_unit" in unit_field: - field = unit_field.rsplit("_unit", 1)[0] - value_fields[field] = (unit_field, unit_value) - - for (field, value) in values: - if field in value_fields.keys(): - # matching value for a unit that we have - if value and not value_fields[field][1]: - # validation error - raise ValueError(f"{field}_unit is required if {field} is provided.") + # Accumulate a dictionary mapping variable : unit/unit_value + for (unit_name, unit_value) in values: + if "_unit" in unit_name and not unit_value: + var_name = unit_name.rsplit('_unit', 1)[0] + + # Go through all the values again, if any value matches the variable name + # and is set, then the unit needs to be set as well + for (variable_name, variable_value) in values: + if variable_name == var_name: + if variable_value: + raise ValueError( + f"Unit {unit_name} is required when {variable_name} is set." + ) + # if we found our variable and it was None, we can move on + continue + + # One more time, now looking for the multi-variable condition + for (variable_name, variable_value) in values: + # skip the unit itself + if variable_name is not unit_name: + if var_name in variable_name and variable_value: + raise ValueError( + f"Unit {unit_name} is required when {variable_name} is set." + ) return values diff --git a/tests/test_base.py b/tests/test_base.py index d122e5bfa..1487c0c79 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -5,9 +5,10 @@ from pathlib import Path from unittest.mock import MagicMock, call, mock_open, patch -from pydantic import create_model +from pydantic import create_model, ValidationError, Field +from typing import Optional -from aind_data_schema.base import AwareDatetimeWithDefault +from aind_data_schema.base import AwareDatetimeWithDefault, AindModel from aind_data_schema.core.subject import Subject @@ -55,6 +56,50 @@ def test_aware_datetime_with_setting(self): expected_json = '{"dt":"2020-10-10T01:02:03Z"}' self.assertEqual(expected_json, model_instance.model_dump_json()) + def test_units(self): + """Test that models with value/value_unit pairs throw errors properly""" + + class TestModel(AindModel): + value: Optional[str] = Field(default=None) + value_unit: Optional[str] = Field(default=None) + + self.assertRaises(ValidationError, lambda: TestModel( + value="value" + )) + + test0 = TestModel( + value="value", + value_unit="unit" + ) + self.assertIsNotNone(test0) + + # it's fine if units are set and the value isn't + test1 = TestModel( + value_unit="unit" + ) + self.assertIsNotNone(test1) + + # Multi-unit condition + class MultiModel(AindModel): + value_one: Optional[str] = Field(default=None) + value_two: Optional[str] = Field(default=None) + value_unit: Optional[str] = Field(default=None) + + self.assertRaises(ValidationError, lambda: MultiModel( + value_one="value" + )) + + test2 = MultiModel( + value_one="value1", + value_unit="unit" + ) + self.assertIsNotNone(test2) + + test3 = MultiModel( + value_unit="unit" + ) + self.assertIsNotNone(test3) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_processing.py b/tests/test_processing.py index 0f5952c8d..d0fa1be3d 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -94,7 +94,7 @@ def test_resource_usage_unit_validators(self): ram=1, ) - expected_exception = "ram_unit is required if ram is provided" + expected_exception = "Unit ram_unit is required when ram is set" self.assertTrue(expected_exception in repr(e.exception)) @@ -116,7 +116,7 @@ def test_resource_usage_unit_validators(self): system_memory=1, ) - expected_exception = "system_memory_unit is required if system_memory is provided" + expected_exception = "Unit system_memory_unit is required when system_memory is set" self.assertTrue(expected_exception in repr(e.exception)) From 352315df4c7cad5837d4e152f68d20f594ffdf9a Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 21 Oct 2024 20:08:36 -0700 Subject: [PATCH 03/23] fix: fix an issue with name parsing --- src/aind_data_schema/base.py | 3 ++- tests/test_base.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index 9f606e8e2..e020cbf69 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -61,6 +61,7 @@ def unit_validator(cls, values): for (unit_name, unit_value) in values: if "_unit" in unit_name and not unit_value: var_name = unit_name.rsplit('_unit', 1)[0] + root_name = unit_name.split('_')[0] # Go through all the values again, if any value matches the variable name # and is set, then the unit needs to be set as well @@ -77,7 +78,7 @@ def unit_validator(cls, values): for (variable_name, variable_value) in values: # skip the unit itself if variable_name is not unit_name: - if var_name in variable_name and variable_value: + if root_name in variable_name and variable_value: raise ValueError( f"Unit {unit_name} is required when {variable_name} is set." ) diff --git a/tests/test_base.py b/tests/test_base.py index 1487c0c79..da2c4a3ec 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -81,16 +81,16 @@ class TestModel(AindModel): # Multi-unit condition class MultiModel(AindModel): - value_one: Optional[str] = Field(default=None) - value_two: Optional[str] = Field(default=None) + value_one_with_depth: Optional[str] = Field(default=None) + value_two_with_depth: Optional[str] = Field(default=None) value_unit: Optional[str] = Field(default=None) self.assertRaises(ValidationError, lambda: MultiModel( - value_one="value" + value_one_with_depth="value" )) test2 = MultiModel( - value_one="value1", + value_one_with_depth="value1", value_unit="unit" ) self.assertIsNotNone(test2) From 8ba80c687897d6c0d2e4778364b27282ce2cc3ad Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 24 Oct 2024 23:57:35 -0400 Subject: [PATCH 04/23] docs: adding Unit doc information to contributing --- CONTRIBUTING.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38df76c04..8ca7f3d12 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,10 +80,29 @@ There are several libraries used to run linters and check documentation. We've i ``` **NOTE**: Please note that these linters are automatically run in github actions when a PR is opened. These linters must pass for a PR to merge. +### Units +Unit types (i.e. anything from [aind_data_schema_models.units](https://github.com/AllenNeuralDynamics/aind-data-schema-models/blob/main/src/aind_data_schema_models/units.py)) should always be paired with a variable in one of two patterns. + +When you have a single `variable` with a unit, you should add the `_unit` suffix on the name of the unit: + +```python +variable: type = Field(...) +variable_unit: XUnit = Field(...) +``` + +If the variable is `Optional[]` the unit should also be marked as optional. + +If you have multiple variables that map onto a single unit type, start each `variable` with the same prefix, i.e. The prefix should be unique within the class and any class that is inherited. + +```python +variable_first_example: type = Field(...) +variable_second_example: type = Field(...) +variable_unit: type = Field(...) +``` + ## Documentation and Style Guide Documentation is required for contributing to this project. We have settled on using Numpy's conventions as a default: [Numpy docstring standards](https://numpydoc.readthedocs.io/en/latest/format.html) - ## Pull Requests For internal members, please create a branch. For external members, please fork the repo and open a pull request from the fork. We'll primarily use [Angular](https://github.com/angular/angular/blob/main/CONTRIBUTING.md#commit) style for commit messages. Roughly, they should follow the pattern: ``` From 4f010d2c8e478ee4e42029449df4b6b9d0222bcb Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 24 Oct 2024 23:58:55 -0400 Subject: [PATCH 05/23] refactor: making units optional where possible --- src/aind_data_schema/components/devices.py | 15 +++++++-------- src/aind_data_schema/components/tile.py | 2 +- src/aind_data_schema/core/procedures.py | 10 +++++----- src/aind_data_schema/core/session.py | 14 +++++++------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/aind_data_schema/components/devices.py b/src/aind_data_schema/components/devices.py index 2f6bffac8..38747ec77 100644 --- a/src/aind_data_schema/components/devices.py +++ b/src/aind_data_schema/components/devices.py @@ -317,7 +317,7 @@ class Detector(Device): cooling: Cooling = Field(default=Cooling.NONE, title="Cooling") computer_name: Optional[str] = Field(default=None, title="Name of computer receiving data from this camera") frame_rate: Optional[Decimal] = Field(default=None, title="Frame rate (Hz)", description="Frame rate being used") - frame_rate_unit: FrequencyUnit = Field(default=FrequencyUnit.HZ, title="Frame rate unit") + frame_rate_unit: Optional[FrequencyUnit] = Field(default=None, title="Frame rate unit") immersion: Optional[ImmersionMedium] = Field(default=None, title="Immersion") chroma: Optional[CameraChroma] = Field(default=None, title="Camera chroma") sensor_width: Optional[int] = Field(default=None, title="Width of the sensor (pixels)") @@ -382,7 +382,7 @@ class Filter(Device): height: Optional[Decimal] = Field(default=None, title="Height (mm)") size_unit: SizeUnit = Field(default=SizeUnit.MM, title="Size unit") thickness: Optional[Decimal] = Field(default=None, title="Thickness (mm)", ge=0) - thickness_unit: SizeUnit = Field(default=SizeUnit.MM, title="Thickness unit") + thickness_unit: Optional[SizeUnit] = Field(default=None, title="Thickness unit") filter_wheel_index: Optional[int] = Field(default=None, title="Filter wheel index") cut_off_wavelength: Optional[int] = Field(default=None, title="Cut-off wavelength (nm)") cut_on_wavelength: Optional[int] = Field(default=None, title="Cut-on wavelength (nm)") @@ -405,7 +405,7 @@ class Lens(Device): # optional fields focal_length: Optional[Decimal] = Field(default=None, title="Focal length of the lens (mm)") - focal_length_unit: SizeUnit = Field(default=SizeUnit.MM, title="Focal length unit") + focal_length_unit: Optional[SizeUnit] = Field(default=None, title="Focal length unit") size: Optional[LensSize] = Field(default=None, title="Size (inches)") lens_size_unit: SizeUnit = Field(default=SizeUnit.IN, title="Lens size unit") optimized_wavelength_range: Optional[str] = Field(default=None, title="Optimized wavelength range (nm)") @@ -469,7 +469,7 @@ class DAQChannel(AindModel): port: Optional[int] = Field(default=None, title="DAQ port") channel_index: Optional[int] = Field(default=None, title="DAQ channel index") sample_rate: Optional[Decimal] = Field(default=None, title="DAQ channel sample rate (Hz)") - sample_rate_unit: FrequencyUnit = Field(default=FrequencyUnit.HZ, title="Sample rate unit") + sample_rate_unit: Optional[FrequencyUnit] = Field(default=None, title="Sample rate unit") event_based_sampling: Optional[bool] = Field( default=None, title="Set to true if DAQ channel is sampled at irregular intervals" ) @@ -545,7 +545,7 @@ class LightEmittingDiode(Device): wavelength: int = Field(..., title="Wavelength (nm)") wavelength_unit: SizeUnit = Field(default=SizeUnit.NM, title="Wavelength unit") bandwidth: Optional[int] = Field(default=None, title="Bandwidth (FWHM)") - bandwidth_unit: SizeUnit = Field(default=SizeUnit.NM, title="Bandwidth unit") + bandwidth_unit: Optional[SizeUnit] = Field(default=None, title="Bandwidth unit") class Lamp(Device): @@ -556,7 +556,7 @@ class Lamp(Device): wavelength_max: Optional[int] = Field(default=None, title="Wavelength maximum (nm)") wavelength_unit: SizeUnit = Field(default=SizeUnit.NM, title="Wavelength unit") temperature: Optional[int] = Field(default=None, title="Temperature (K)") - temperature_unit: TemperatureUnit = Field(default=TemperatureUnit.K, title="Temperature unit") + temperature_unit: Optional[TemperatureUnit] = Field(default=None, title="Temperature unit") class ProbePort(AindModel): @@ -650,7 +650,6 @@ class FiberProbe(Device): device_type: Literal["Fiber optic probe"] = "Fiber optic probe" core_diameter: Decimal = Field(..., title="Core diameter (um)") - # TODO: Check if this should be an enum? core_diameter_unit: SizeUnit = Field(default=SizeUnit.UM, title="Core diameter unit") numerical_aperture: Decimal = Field(..., title="Numerical aperture") ferrule_material: Optional[FerruleMaterial] = Field(default=None, title="Ferrule material") @@ -707,7 +706,7 @@ class PockelsCell(Device): off_time: Optional[Decimal] = Field(default=None, title="Off time (fraction of cycle)") time_setting_unit: UnitlessUnit = Field(default=UnitlessUnit.FC, title="Time setting unit") beam_modulation: Optional[Decimal] = Field(default=None, title="Beam modulation (V)") - beam_modulation_unit: VoltageUnit = Field(default=VoltageUnit.V, title="Beam modulation unit") + beam_modulation_unit: Optional[VoltageUnit] = Field(default=None, title="Beam modulation unit") class Enclosure(Device): diff --git a/src/aind_data_schema/components/tile.py b/src/aind_data_schema/components/tile.py index 3ffe8a444..6a47974a8 100644 --- a/src/aind_data_schema/components/tile.py +++ b/src/aind_data_schema/components/tile.py @@ -32,7 +32,7 @@ class Channel(AindModel): filter_wheel_index: int = Field(..., title="Filter wheel index") # dilation dilation: Optional[int] = Field(default=None, title="Dilation (pixels)") - dilation_unit: SizeUnit = Field(default=SizeUnit.PX, title="Dilation unit") + dilation_unit: Optional[SizeUnit] = Field(default=None, title="Dilation unit") description: Optional[str] = Field(default=None, title="Description") diff --git a/src/aind_data_schema/core/procedures.py b/src/aind_data_schema/core/procedures.py index 9e9cda4fd..dbb901fe0 100644 --- a/src/aind_data_schema/core/procedures.py +++ b/src/aind_data_schema/core/procedures.py @@ -306,7 +306,7 @@ class Craniotomy(AindModel): dura_removed: Optional[bool] = Field(default=None, title="Dura removed") protective_material: Optional[ProtectiveMaterial] = Field(default=None, title="Protective material") recovery_time: Optional[Decimal] = Field(default=None, title="Recovery time") - recovery_time_unit: TimeUnit = Field(default=TimeUnit.M, title="Recovery time unit") + recovery_time_unit: Optional[TimeUnit] = Field(default=None, title="Recovery time unit") class Headframe(AindModel): @@ -332,7 +332,7 @@ class ProtectiveMaterialReplacement(AindModel): ground_wire_hole: Optional[int] = Field(default=None, title="Ground wire hole") ground_wire_material: Optional[GroundWireMaterial] = Field(default=None, title="Ground wire material") ground_wire_diameter: Optional[Decimal] = Field(default=None, title="Ground wire diameter") - ground_wire_diameter_unit: SizeUnit = Field(default=SizeUnit.IN, title="Ground wire diameter unit") + ground_wire_diameter_unit: Optional[SizeUnit] = Field(default=None, title="Ground wire diameter unit") well_part_number: Optional[str] = Field(default=None, title="Well part number") well_type: Optional[str] = Field(default=None, title="Well type") @@ -384,7 +384,7 @@ class NonViralMaterial(Reagent): concentration: Optional[Decimal] = Field( default=None, title="Concentration", description="Must provide concentration unit" ) - concentration_unit: str = Field(default="mg/mL", title="Concentration unit") + concentration_unit: Optional[str] = Field(default=None, title="Concentration unit", description="For example, mg/mL") class Injection(AindModel): @@ -394,9 +394,9 @@ class Injection(AindModel): Annotated[Union[ViralMaterial, NonViralMaterial], Field(..., discriminator="material_type")] ] = Field(..., title="Injection material", min_length=1) recovery_time: Optional[Decimal] = Field(default=None, title="Recovery time") - recovery_time_unit: TimeUnit = Field(default=TimeUnit.M, title="Recovery time unit") + recovery_time_unit: Optional[TimeUnit] = Field(default=None, title="Recovery time unit") injection_duration: Optional[Decimal] = Field(default=None, title="Injection duration") - injection_duration_unit: TimeUnit = Field(default=TimeUnit.M, title="Injection duration unit") + injection_duration_unit: Optional[TimeUnit] = Field(default=None, title="Injection duration unit") instrument_id: Optional[str] = Field(default=None, title="Instrument ID") protocol_id: str = Field(..., title="Protocol ID", description="DOI for protocols.io") diff --git a/src/aind_data_schema/core/session.py b/src/aind_data_schema/core/session.py index d5d3fef35..6901f6924 100644 --- a/src/aind_data_schema/core/session.py +++ b/src/aind_data_schema/core/session.py @@ -86,7 +86,7 @@ class LightEmittingDiodeConfig(AindModel): device_type: Literal["Light emitting diode"] = "Light emitting diode" name: str = Field(..., title="Name") excitation_power: Optional[Decimal] = Field(default=None, title="Excitation power (mW)") - excitation_power_unit: PowerUnit = Field(default=PowerUnit.MW, title="Excitation power unit") + excitation_power_unit: Optional[PowerUnit] = Field(default=None, title="Excitation power unit") class FieldOfView(AindModel): @@ -111,20 +111,20 @@ class FieldOfView(AindModel): fov_scale_factor: Decimal = Field(..., title="FOV scale factor (um/pixel)") fov_scale_factor_unit: str = Field(default="um/pixel", title="FOV scale factor unit") frame_rate: Optional[Decimal] = Field(default=None, title="Frame rate (Hz)") - frame_rate_unit: FrequencyUnit = Field(default=FrequencyUnit.HZ, title="Frame rate unit") + frame_rate_unit: Optional[FrequencyUnit] = Field(default=None, title="Frame rate unit") coupled_fov_index: Optional[int] = Field( default=None, title="Coupled FOV", description="Coupled planes for multiscope" ) power: Optional[Decimal] = Field( default=None, title="Power", description="For coupled planes, this power is shared by both planes" ) - power_unit: PowerUnit = Field(default=PowerUnit.PERCENT, title="Power unit") + power_unit: Optional[PowerUnit] = Field(default=None, title="Power unit") power_ratio: Optional[Decimal] = Field(default=None, title="Power ratio for coupled planes") scanfield_z: Optional[int] = Field( default=None, title="Z stage position of the fastz actuator for a given targeted depth", ) - scanfield_z_unit: SizeUnit = Field(default=SizeUnit.UM, title="Z stage position unit") + scanfield_z_unit: Optional[SizeUnit] = Field(default=None, title="Z stage position unit") scanimage_roi_index: Optional[int] = Field(default=None, title="ScanImage ROI index") notes: Optional[str] = Field(default=None, title="Notes") @@ -222,7 +222,7 @@ class ManipulatorModule(DomeModule): Field(default=None, title="Anatomical coordinate reference") ) surface_z: Optional[Decimal] = Field(default=None, title="Surface z") - surface_z_unit: SizeUnit = Field(default=SizeUnit.UM, title="Surface z unit") + surface_z_unit: Optional[SizeUnit] = Field(default=None, title="Surface z unit") dye: Optional[str] = Field(default=None, title="Dye") implant_hole_number: Optional[int] = Field(default=None, title="Implant hole number") @@ -241,7 +241,7 @@ class LaserConfig(AindModel): wavelength: int = Field(..., title="Wavelength (nm)") wavelength_unit: SizeUnit = Field(default=SizeUnit.NM, title="Wavelength unit") excitation_power: Optional[Decimal] = Field(default=None, title="Excitation power (mW)") - excitation_power_unit: PowerUnit = Field(default=PowerUnit.MW, title="Excitation power unit") + excitation_power_unit: Optional[PowerUnit] = Field(default=None, title="Excitation power unit") LIGHT_SOURCE_CONFIGS = Annotated[ @@ -293,7 +293,7 @@ class SpeakerConfig(AindModel): name: str = Field(..., title="Name", description="Must match rig json") volume: Optional[Decimal] = Field(default=None, title="Volume (dB)") - volume_unit: SoundIntensityUnit = Field(default=SoundIntensityUnit.DB, title="Volume unit") + volume_unit: Optional[SoundIntensityUnit] = Field(default=None, title="Volume unit") # MRI components From a3c7c430c9cadbd7d932418428bef6a6221522a6 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 24 Oct 2024 23:59:57 -0400 Subject: [PATCH 06/23] chore: lint --- src/aind_data_schema/base.py | 18 +++++++---------- src/aind_data_schema/core/procedures.py | 4 +++- src/aind_data_schema/core/processing.py | 2 +- tests/test_base.py | 26 ++++++------------------- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index e020cbf69..81f7e722f 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -58,30 +58,26 @@ def unit_validator(cls, values): if any of variable_* are set """ # Accumulate a dictionary mapping variable : unit/unit_value - for (unit_name, unit_value) in values: + for unit_name, unit_value in values: if "_unit" in unit_name and not unit_value: - var_name = unit_name.rsplit('_unit', 1)[0] - root_name = unit_name.split('_')[0] + var_name = unit_name.rsplit("_unit", 1)[0] + root_name = unit_name.split("_")[0] # Go through all the values again, if any value matches the variable name # and is set, then the unit needs to be set as well - for (variable_name, variable_value) in values: + for variable_name, variable_value in values: if variable_name == var_name: if variable_value: - raise ValueError( - f"Unit {unit_name} is required when {variable_name} is set." - ) + raise ValueError(f"Unit {unit_name} is required when {variable_name} is set.") # if we found our variable and it was None, we can move on continue # One more time, now looking for the multi-variable condition - for (variable_name, variable_value) in values: + for variable_name, variable_value in values: # skip the unit itself if variable_name is not unit_name: if root_name in variable_name and variable_value: - raise ValueError( - f"Unit {unit_name} is required when {variable_name} is set." - ) + raise ValueError(f"Unit {unit_name} is required when {variable_name} is set.") return values diff --git a/src/aind_data_schema/core/procedures.py b/src/aind_data_schema/core/procedures.py index dbb901fe0..37288ebf5 100644 --- a/src/aind_data_schema/core/procedures.py +++ b/src/aind_data_schema/core/procedures.py @@ -384,7 +384,9 @@ class NonViralMaterial(Reagent): concentration: Optional[Decimal] = Field( default=None, title="Concentration", description="Must provide concentration unit" ) - concentration_unit: Optional[str] = Field(default=None, title="Concentration unit", description="For example, mg/mL") + concentration_unit: Optional[str] = Field( + default=None, title="Concentration unit", description="For example, mg/mL" + ) class Injection(AindModel): diff --git a/src/aind_data_schema/core/processing.py b/src/aind_data_schema/core/processing.py index 5ac811c7a..7b6da9fbe 100644 --- a/src/aind_data_schema/core/processing.py +++ b/src/aind_data_schema/core/processing.py @@ -5,7 +5,7 @@ from aind_data_schema_models.process_names import ProcessName from aind_data_schema_models.units import MemoryUnit, UnitlessUnit -from pydantic import Field, ValidationInfo, field_validator, model_validator +from pydantic import Field, ValidationInfo, field_validator from aind_data_schema.base import AindCoreModel, AindGeneric, AindGenericType, AindModel, AwareDatetimeWithDefault from aind_data_schema.components.tile import Tile diff --git a/tests/test_base.py b/tests/test_base.py index da2c4a3ec..84da7b7f2 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -63,20 +63,13 @@ class TestModel(AindModel): value: Optional[str] = Field(default=None) value_unit: Optional[str] = Field(default=None) - self.assertRaises(ValidationError, lambda: TestModel( - value="value" - )) + self.assertRaises(ValidationError, lambda: TestModel(value="value")) - test0 = TestModel( - value="value", - value_unit="unit" - ) + test0 = TestModel(value="value", value_unit="unit") self.assertIsNotNone(test0) # it's fine if units are set and the value isn't - test1 = TestModel( - value_unit="unit" - ) + test1 = TestModel(value_unit="unit") self.assertIsNotNone(test1) # Multi-unit condition @@ -85,19 +78,12 @@ class MultiModel(AindModel): value_two_with_depth: Optional[str] = Field(default=None) value_unit: Optional[str] = Field(default=None) - self.assertRaises(ValidationError, lambda: MultiModel( - value_one_with_depth="value" - )) + self.assertRaises(ValidationError, lambda: MultiModel(value_one_with_depth="value")) - test2 = MultiModel( - value_one_with_depth="value1", - value_unit="unit" - ) + test2 = MultiModel(value_one_with_depth="value1", value_unit="unit") self.assertIsNotNone(test2) - test3 = MultiModel( - value_unit="unit" - ) + test3 = MultiModel(value_unit="unit") self.assertIsNotNone(test3) From 0af50bff3c4513d59dcbee04c737948314427c90 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 25 Oct 2024 00:09:10 -0400 Subject: [PATCH 07/23] test/docs: docstring fixes, changing names --- tests/test_base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_base.py b/tests/test_base.py index 84da7b7f2..51f3eb6c8 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,4 +1,4 @@ -""" tests for Subject """ +""" tests for base """ import unittest from datetime import datetime, timezone @@ -60,6 +60,7 @@ def test_units(self): """Test that models with value/value_unit pairs throw errors properly""" class TestModel(AindModel): + """temporary test model""" value: Optional[str] = Field(default=None) value_unit: Optional[str] = Field(default=None) @@ -74,9 +75,10 @@ class TestModel(AindModel): # Multi-unit condition class MultiModel(AindModel): - value_one_with_depth: Optional[str] = Field(default=None) - value_two_with_depth: Optional[str] = Field(default=None) - value_unit: Optional[str] = Field(default=None) + """temporary test model with multiple variables""" + value_multi_one_with_depth: Optional[str] = Field(default=None) + value_multi_two_with_depth: Optional[str] = Field(default=None) + value_multi_unit: Optional[str] = Field(default=None) self.assertRaises(ValidationError, lambda: MultiModel(value_one_with_depth="value")) From a5e8212a1f5e424d094f4f0a78a79cdd1a9d5ee9 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 25 Oct 2024 00:09:25 -0400 Subject: [PATCH 08/23] fix: fixing multi variable to allow longer prefix names --- src/aind_data_schema/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index 81f7e722f..6157329e9 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -61,7 +61,6 @@ def unit_validator(cls, values): for unit_name, unit_value in values: if "_unit" in unit_name and not unit_value: var_name = unit_name.rsplit("_unit", 1)[0] - root_name = unit_name.split("_")[0] # Go through all the values again, if any value matches the variable name # and is set, then the unit needs to be set as well @@ -75,8 +74,8 @@ def unit_validator(cls, values): # One more time, now looking for the multi-variable condition for variable_name, variable_value in values: # skip the unit itself - if variable_name is not unit_name: - if root_name in variable_name and variable_value: + if var_name is not unit_name: + if var_name in variable_name and variable_value: raise ValueError(f"Unit {unit_name} is required when {variable_name} is set.") return values From c76e2c5aae37fff33aae1d9f447a6a44ddd7725d Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 31 Oct 2024 22:14:02 -0700 Subject: [PATCH 09/23] docs: improved example --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ca7f3d12..13dc073f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,12 +92,12 @@ variable_unit: XUnit = Field(...) If the variable is `Optional[]` the unit should also be marked as optional. -If you have multiple variables that map onto a single unit type, start each `variable` with the same prefix, i.e. The prefix should be unique within the class and any class that is inherited. +If you have multiple variables that map onto a single unit type, start each `variable` with the same prefix. The prefix should be unique within the class (watch out for inherited fields). ```python -variable_first_example: type = Field(...) -variable_second_example: type = Field(...) -variable_unit: type = Field(...) +fov_width: type = Field(...) +fov_height: type = Field(...) +fov_unit: XUnit = Field(...) ``` ## Documentation and Style Guide From 16ffb3b2d05494791085283436dd812d659c6fed Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 31 Oct 2024 22:15:51 -0700 Subject: [PATCH 10/23] chore: lint --- src/aind_data_schema/base.py | 1 - tests/test_metadata.py | 5 ++--- tests/test_processing.py | 9 ++++----- tests/test_quality_control.py | 2 +- tests/test_rig.py | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index 2b611b49f..24da5fd22 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -18,7 +18,6 @@ model_validator, ) from pydantic.functional_validators import WrapValidator -from pydantic import model_validator from typing_extensions import Annotated diff --git a/tests/test_metadata.py b/tests/test_metadata.py index cadd1bde2..fd6ed0432 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -5,9 +5,9 @@ import unittest from datetime import time +from aind_data_schema_models.modalities import Modality from aind_data_schema_models.organizations import Organization from aind_data_schema_models.platforms import Platform -from aind_data_schema_models.modalities import Modality from pydantic import ValidationError from pydantic import __version__ as pyd_version @@ -327,8 +327,7 @@ def test_validate_rig_session_compatibility(self): ) def test_validate_old_schema_version(self): - """Tests that old schema versions are ignored during validation - """ + """Tests that old schema versions are ignored during validation""" m = Metadata.model_construct( name="name", location="location", diff --git a/tests/test_processing.py b/tests/test_processing.py index d0fa1be3d..2230b306c 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -2,20 +2,19 @@ import re import unittest +from datetime import datetime import pydantic - -from datetime import datetime +from aind_data_schema_models.system_architecture import CPUArchitecture, OperatingSystem +from aind_data_schema_models.units import MemoryUnit from aind_data_schema.core.processing import ( DataProcess, PipelineProcess, Processing, - ResourceUsage, ResourceTimestamped, + ResourceUsage, ) -from aind_data_schema_models.system_architecture import OperatingSystem, CPUArchitecture -from aind_data_schema_models.units import MemoryUnit PYD_VERSION = re.match(r"(\d+.\d+).\d+", pydantic.__version__).group(1) diff --git a/tests/test_quality_control.py b/tests/test_quality_control.py index 4da9186a0..5ea35b5df 100644 --- a/tests/test_quality_control.py +++ b/tests/test_quality_control.py @@ -6,7 +6,7 @@ from aind_data_schema_models.modalities import Modality from pydantic import ValidationError -from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status, QCStatus +from aind_data_schema.core.quality_control import QCEvaluation, QCMetric, QCStatus, QualityControl, Stage, Status class QualityControlTests(unittest.TestCase): diff --git a/tests/test_rig.py b/tests/test_rig.py index 32b243125..0eed7daae 100644 --- a/tests/test_rig.py +++ b/tests/test_rig.py @@ -1,7 +1,7 @@ """ test Rig """ -import unittest import json +import unittest from datetime import date, datetime from aind_data_schema_models.modalities import Modality From 9551a3b728228e2a91f1ba01e66896919a52ce47 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 31 Oct 2024 22:32:29 -0700 Subject: [PATCH 11/23] tests: update examples --- examples/aibs_smartspim_instrument.json | 2 +- examples/aibs_smartspim_instrument.py | 4 +++ examples/aind_smartspim_instrument.json | 2 +- examples/aind_smartspim_instrument.py | 7 +++++ examples/bergamo_ophys_session.json | 4 +-- examples/bergamo_ophys_session.py | 2 ++ examples/ephys_rig.json | 18 +++++------ examples/ephys_rig.py | 9 +++++- examples/ephys_session.json | 8 ++--- examples/exaspim_acquisition.json | 4 +-- examples/exaspim_instrument.json | 2 +- examples/exaspim_instrument.py | 8 +++++ examples/fip_behavior_rig.json | 42 ++++++++++++------------- examples/fip_behavior_rig.py | 4 +++ examples/fip_ophys_rig.json | 42 ++++++++++++------------- examples/fip_ophys_rig.py | 4 +++ examples/multiplane_ophys_session.py | 27 ++++++++++++++-- examples/ophys_procedures.json | 4 +-- examples/procedures.json | 6 ++-- examples/quality_control.json | 3 -- 20 files changed, 129 insertions(+), 73 deletions(-) diff --git a/examples/aibs_smartspim_instrument.json b/examples/aibs_smartspim_instrument.json index 1b78e8ae0..5a7de8d10 100644 --- a/examples/aibs_smartspim_instrument.json +++ b/examples/aibs_smartspim_instrument.json @@ -84,7 +84,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": null, "chroma": null, "sensor_width": null, diff --git a/examples/aibs_smartspim_instrument.py b/examples/aibs_smartspim_instrument.py index d2de6a26c..87519b5eb 100644 --- a/examples/aibs_smartspim_instrument.py +++ b/examples/aibs_smartspim_instrument.py @@ -3,6 +3,7 @@ import datetime from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import SizeUnit from aind_data_schema.components.devices import ( AdditionalImagingDevice, @@ -166,6 +167,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF03-525/50-25", filter_wheel_index=0, ), @@ -175,6 +177,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF01-600/52-25", filter_wheel_index=1, ), @@ -184,6 +187,7 @@ manufacturer=Organization.CHROMA, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="ET690/50m", filter_wheel_index=2, ), diff --git a/examples/aind_smartspim_instrument.json b/examples/aind_smartspim_instrument.json index aebce51af..5decea225 100644 --- a/examples/aind_smartspim_instrument.json +++ b/examples/aind_smartspim_instrument.json @@ -104,7 +104,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": null, "chroma": null, "sensor_width": null, diff --git a/examples/aind_smartspim_instrument.py b/examples/aind_smartspim_instrument.py index b78713267..74673641c 100644 --- a/examples/aind_smartspim_instrument.py +++ b/examples/aind_smartspim_instrument.py @@ -3,6 +3,7 @@ import datetime from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import SizeUnit from aind_data_schema.components.devices import Filter, Laser, MotorizedStage, OpticalTable, ScanningStage from aind_data_schema.core.instrument import Com, Detector, Instrument, Objective @@ -106,6 +107,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF01-469/35-25", filter_wheel_index=0, serial_number="Unknown-0", @@ -116,6 +118,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF01-525/45-25", filter_wheel_index=1, serial_number="Unknown-1", @@ -126,6 +129,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF01-593/40-25", filter_wheel_index=2, serial_number="Unknown-2", @@ -136,6 +140,7 @@ manufacturer=Organization.SEMROCK, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FF01-624/40-25", filter_wheel_index=3, serial_number="Unknown-3", @@ -146,6 +151,7 @@ manufacturer=Organization.CHROMA, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="ET667/30m", filter_wheel_index=4, serial_number="Unknown-4", @@ -156,6 +162,7 @@ manufacturer=Organization.THORLABS, diameter=25, thickness=2.0, + thickness_unit=SizeUnit.MM, model="FELH0700", filter_wheel_index=5, serial_number="Unknown-5", diff --git a/examples/bergamo_ophys_session.json b/examples/bergamo_ophys_session.json index e649d8d6a..e01402234 100644 --- a/examples/bergamo_ophys_session.json +++ b/examples/bergamo_ophys_session.json @@ -68,10 +68,10 @@ "frame_rate_unit": "hertz", "coupled_fov_index": null, "power": null, - "power_unit": "percent", + "power_unit": null, "power_ratio": null, "scanfield_z": null, - "scanfield_z_unit": "micrometer", + "scanfield_z_unit": null, "scanimage_roi_index": null, "notes": null } diff --git a/examples/bergamo_ophys_session.py b/examples/bergamo_ophys_session.py index f750687a8..0e8bccd50 100644 --- a/examples/bergamo_ophys_session.py +++ b/examples/bergamo_ophys_session.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.units import FrequencyUnit from aind_data_schema.components.stimulus import PhotoStimulation, PhotoStimulationGroup from aind_data_schema.core.session import ( @@ -64,6 +65,7 @@ magnification="1x", fov_scale_factor=1.5, frame_rate=20, + frame_rate_unit=FrequencyUnit.HZ ), ], ), diff --git a/examples/ephys_rig.json b/examples/ephys_rig.json index 02b9e0b6a..3a1f25ec1 100644 --- a/examples/ephys_rig.json +++ b/examples/ephys_rig.json @@ -123,7 +123,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -232,7 +232,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -386,7 +386,7 @@ "additional_settings": {}, "notes": null, "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -464,7 +464,7 @@ "additional_settings": {}, "notes": null, "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -542,7 +542,7 @@ "additional_settings": {}, "notes": null, "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -620,7 +620,7 @@ "additional_settings": {}, "notes": null, "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -815,7 +815,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -825,7 +825,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -835,7 +835,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null } ], diff --git a/examples/ephys_rig.py b/examples/ephys_rig.py index 07441ff36..51aa6a618 100644 --- a/examples/ephys_rig.py +++ b/examples/ephys_rig.py @@ -5,6 +5,7 @@ from aind_data_schema_models.harp_types import HarpDeviceType from aind_data_schema_models.modalities import Modality from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import FrequencyUnit, SizeUnit from aind_data_schema.components.devices import ( Calibration, @@ -91,6 +92,7 @@ manufacturer=Organization.FLIR, computer_name=ephys_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -105,6 +107,7 @@ manufacturer=Organization.FLIR, computer_name=ephys_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -119,6 +122,7 @@ manufacturer=Organization.FLIR, computer_name=ephys_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -133,6 +137,7 @@ manufacturer=Organization.FLIR, computer_name=ephys_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -197,7 +202,7 @@ description="850 nm longpass filter", ) -lens = Lens(name="Camera lens", focal_length=15, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") +lens = Lens(name="Camera lens", focal_length=15, focal_length_unit=SizeUnit.MM, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") face_camera = Camera( name="Face Camera", @@ -206,6 +211,7 @@ manufacturer=Organization.FLIR, computer_name=behavior_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -228,6 +234,7 @@ manufacturer=Organization.FLIR, computer_name=behavior_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", diff --git a/examples/ephys_session.json b/examples/ephys_session.json index 48eb28aa8..85991e0ff 100644 --- a/examples/ephys_session.json +++ b/examples/ephys_session.json @@ -57,7 +57,7 @@ "anatomical_coordinates": null, "anatomical_reference": null, "surface_z": null, - "surface_z_unit": "micrometer", + "surface_z_unit": null, "dye": null, "implant_hole_number": null }, @@ -90,7 +90,7 @@ "anatomical_coordinates": null, "anatomical_reference": null, "surface_z": null, - "surface_z_unit": "micrometer", + "surface_z_unit": null, "dye": null, "implant_hole_number": null } @@ -192,7 +192,7 @@ "anatomical_coordinates": null, "anatomical_reference": null, "surface_z": null, - "surface_z_unit": "micrometer", + "surface_z_unit": null, "dye": null, "implant_hole_number": null }, @@ -225,7 +225,7 @@ "anatomical_coordinates": null, "anatomical_reference": null, "surface_z": null, - "surface_z_unit": "micrometer", + "surface_z_unit": null, "dye": null, "implant_hole_number": null } diff --git a/examples/exaspim_acquisition.json b/examples/exaspim_acquisition.json index a4117ea15..c7da54cdc 100644 --- a/examples/exaspim_acquisition.json +++ b/examples/exaspim_acquisition.json @@ -97,7 +97,7 @@ "excitation_power_unit": "milliwatt", "filter_wheel_index": 0, "dilation": null, - "dilation_unit": "pixel", + "dilation_unit": null, "description": null }, "notes": "these are my notes", @@ -140,7 +140,7 @@ "excitation_power_unit": "milliwatt", "filter_wheel_index": 0, "dilation": null, - "dilation_unit": "pixel", + "dilation_unit": null, "description": null }, "notes": "these are my notes", diff --git a/examples/exaspim_instrument.json b/examples/exaspim_instrument.json index b46a639f7..76f5cd2d6 100644 --- a/examples/exaspim_instrument.json +++ b/examples/exaspim_instrument.json @@ -81,7 +81,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": null, "chroma": null, "sensor_width": null, diff --git a/examples/exaspim_instrument.py b/examples/exaspim_instrument.py index b4499a747..921855e7c 100644 --- a/examples/exaspim_instrument.py +++ b/examples/exaspim_instrument.py @@ -3,6 +3,7 @@ import datetime from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import SizeUnit, FrequencyUnit from aind_data_schema.components.devices import DAQChannel, DAQDevice, Detector, Filter, Laser from aind_data_schema.core import instrument @@ -79,6 +80,7 @@ manufacturer=Organization.CHROMA, diameter=44.05, thickness=1, + thickness_unit=SizeUnit.MM, model="ZET405/488/561/640mv2", notes="Custom made filter", filter_wheel_index=0, @@ -97,36 +99,42 @@ channel_type="Analog Output", device_name="LAS-08308", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="5", channel_type="Analog Output", device_name="539251", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="4", channel_type="Analog Output", device_name="LAS-08309", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="2", channel_type="Analog Output", device_name="stage-x", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="0", channel_type="Analog Output", device_name="TL-1", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="6", channel_type="Analog Output", device_name="LAS-08307", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), ], ) diff --git a/examples/fip_behavior_rig.json b/examples/fip_behavior_rig.json index 6ce94a51c..eea1e87e9 100644 --- a/examples/fip_behavior_rig.json +++ b/examples/fip_behavior_rig.json @@ -204,7 +204,7 @@ "additional_settings": {}, "notes": "Focal Length 9-22mm 1/3\" IR F1.4", "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -281,7 +281,7 @@ "additional_settings": {}, "notes": "Focal Length 9-22mm 1/3\" IR F1.4", "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -343,7 +343,7 @@ "wavelength": 470, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null }, { "device_type": "Light emitting diode", @@ -366,7 +366,7 @@ "wavelength": 415, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null }, { "device_type": "Light emitting diode", @@ -389,7 +389,7 @@ "wavelength": 565, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null } ], "detectors": [ @@ -416,7 +416,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": "air", "chroma": "Monochrome", "sensor_width": null, @@ -462,7 +462,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": "air", "chroma": "Monochrome", "sensor_width": null, @@ -533,7 +533,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -562,7 +562,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -591,7 +591,7 @@ "height": "25", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 562, "cut_on_wavelength": null, @@ -620,7 +620,7 @@ "height": "24", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -652,7 +652,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -684,7 +684,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -716,7 +716,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -748,7 +748,7 @@ "height": "25.2", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 450, "cut_on_wavelength": null, @@ -780,7 +780,7 @@ "height": "23.2", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 500, "cut_on_wavelength": null, @@ -862,7 +862,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -872,7 +872,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -882,7 +882,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -892,7 +892,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -902,7 +902,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null } ], diff --git a/examples/fip_behavior_rig.py b/examples/fip_behavior_rig.py index 82870e15b..6ce341a96 100644 --- a/examples/fip_behavior_rig.py +++ b/examples/fip_behavior_rig.py @@ -4,6 +4,7 @@ from datetime import date, datetime, timezone from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.units import FrequencyUnit, SizeUnit import aind_data_schema.components.devices as d import aind_data_schema.core.rig as r @@ -26,6 +27,7 @@ data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -55,6 +57,7 @@ data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -301,6 +304,7 @@ model="AC254-080-A-ML", name="Image focusing lens", focal_length=80, + focal_length_unit=SizeUnit.MM, size=1, ) ], diff --git a/examples/fip_ophys_rig.json b/examples/fip_ophys_rig.json index e4b19b2e7..2feddb9ab 100644 --- a/examples/fip_ophys_rig.json +++ b/examples/fip_ophys_rig.json @@ -174,7 +174,7 @@ "additional_settings": {}, "notes": "Focal Length 9-22mm 1/3\" IR F1.4", "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -251,7 +251,7 @@ "additional_settings": {}, "notes": "Focal Length 9-22mm 1/3\" IR F1.4", "focal_length": null, - "focal_length_unit": "millimeter", + "focal_length_unit": null, "size": null, "lens_size_unit": "inch", "optimized_wavelength_range": null, @@ -313,7 +313,7 @@ "wavelength": 470, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null }, { "device_type": "Light emitting diode", @@ -336,7 +336,7 @@ "wavelength": 415, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null }, { "device_type": "Light emitting diode", @@ -359,7 +359,7 @@ "wavelength": 565, "wavelength_unit": "nanometer", "bandwidth": null, - "bandwidth_unit": "nanometer" + "bandwidth_unit": null } ], "detectors": [ @@ -386,7 +386,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": "air", "chroma": "Monochrome", "sensor_width": null, @@ -432,7 +432,7 @@ "cooling": "Air", "computer_name": null, "frame_rate": null, - "frame_rate_unit": "hertz", + "frame_rate_unit": null, "immersion": "air", "chroma": "Monochrome", "sensor_width": null, @@ -503,7 +503,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -532,7 +532,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -561,7 +561,7 @@ "height": "25", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 562, "cut_on_wavelength": null, @@ -590,7 +590,7 @@ "height": "24", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -622,7 +622,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -654,7 +654,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -686,7 +686,7 @@ "height": null, "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": null, "cut_on_wavelength": null, @@ -718,7 +718,7 @@ "height": "25.2", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 450, "cut_on_wavelength": null, @@ -750,7 +750,7 @@ "height": "23.2", "size_unit": "millimeter", "thickness": null, - "thickness_unit": "millimeter", + "thickness_unit": null, "filter_wheel_index": null, "cut_off_wavelength": 500, "cut_on_wavelength": null, @@ -832,7 +832,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -842,7 +842,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -852,7 +852,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -862,7 +862,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null }, { @@ -872,7 +872,7 @@ "port": null, "channel_index": null, "sample_rate": null, - "sample_rate_unit": "hertz", + "sample_rate_unit": null, "event_based_sampling": null } ], diff --git a/examples/fip_ophys_rig.py b/examples/fip_ophys_rig.py index 38bc5c00f..98213e3dc 100644 --- a/examples/fip_ophys_rig.py +++ b/examples/fip_ophys_rig.py @@ -3,6 +3,7 @@ from datetime import date, datetime, timezone from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.units import FrequencyUnit, SizeUnit import aind_data_schema.components.devices as d import aind_data_schema.core.rig as r @@ -25,6 +26,7 @@ data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -54,6 +56,7 @@ data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -237,6 +240,7 @@ model="AC254-080-A-ML", name="Image focusing lens", focal_length=80, + focal_length_unit=SizeUnit.MM, size=1, ) ], diff --git a/examples/multiplane_ophys_session.py b/examples/multiplane_ophys_session.py index 493c197cb..e324fd3c5 100644 --- a/examples/multiplane_ophys_session.py +++ b/examples/multiplane_ophys_session.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from aind_data_schema_models.modalities import Modality -from aind_data_schema_models.units import PowerUnit, SizeUnit +from aind_data_schema_models.units import PowerUnit, SizeUnit, FrequencyUnit from aind_data_schema.core.session import FieldOfView, LaserConfig, Session, Stream @@ -39,7 +39,7 @@ wavelength=920, wavelength_unit="nanometer", excitation_power=10, - excitation_power_unit="milliwatt", + excitation_power_unit=PowerUnit.MW, ), ], ophys_fovs=[ @@ -55,12 +55,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=5, power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=190, targeted_structure="VISp", scanfield_z=230, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=1, ), FieldOfView( @@ -75,11 +77,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=42, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=232, targeted_structure="VISp", scanfield_z=257, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=0, ), FieldOfView( @@ -94,11 +99,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=28, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=136, targeted_structure="VISp", scanfield_z=176, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=3, ), FieldOfView( @@ -113,11 +121,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=28, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=282, targeted_structure="VISp", scanfield_z=307, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=2, ), FieldOfView( @@ -132,11 +143,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=12, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=72, targeted_structure="VISp", scanfield_z=112, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=5, ), FieldOfView( @@ -151,11 +165,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=12, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=326, targeted_structure="VISp", scanfield_z=351, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=4, ), FieldOfView( @@ -170,11 +187,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=5, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=30, targeted_structure="VISp", scanfield_z=70, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=7, ), FieldOfView( @@ -189,11 +209,14 @@ magnification="10x", fov_scale_factor=0.78, frame_rate=9.48, + frame_rate_unit=FrequencyUnit.HZ, power=5, + power_unit=PowerUnit.PERCENT, scanimage_roi_index=0, imaging_depth=364, targeted_structure="VISp", scanfield_z=389, + scanfield_z_unit=SizeUnit.UM, coupled_fov_index=6, ), ], diff --git a/examples/ophys_procedures.json b/examples/ophys_procedures.json index 127b1600c..5d4357de3 100644 --- a/examples/ophys_procedures.json +++ b/examples/ophys_procedures.json @@ -49,9 +49,9 @@ } ], "recovery_time": "0", - "recovery_time_unit": "minute", + "recovery_time_unit": null, "injection_duration": null, - "injection_duration_unit": "minute", + "injection_duration_unit": null, "instrument_id": null, "protocol_id": "5678", "injection_coordinate_ml": "-0.6", diff --git a/examples/procedures.json b/examples/procedures.json index f032e22bb..0f5971161 100644 --- a/examples/procedures.json +++ b/examples/procedures.json @@ -31,7 +31,7 @@ "dura_removed": null, "protective_material": null, "recovery_time": null, - "recovery_time_unit": "minute" + "recovery_time_unit": null }, { "injection_materials": [ @@ -52,9 +52,9 @@ } ], "recovery_time": "0", - "recovery_time_unit": "minute", + "recovery_time_unit": null, "injection_duration": null, - "injection_duration_unit": "minute", + "injection_duration_unit": null, "instrument_id": null, "protocol_id": "5678", "injection_coordinate_ml": "-0.87", diff --git a/examples/quality_control.json b/examples/quality_control.json index 15dfb18a1..72646aab9 100644 --- a/examples/quality_control.json +++ b/examples/quality_control.json @@ -80,7 +80,6 @@ "evaluated_assets": null } ], - "tags": null, "notes": "", "allow_failed_metrics": false }, @@ -122,7 +121,6 @@ "evaluated_assets": null } ], - "tags": null, "notes": "Pass when video_1_num_frames==video_2_num_frames", "allow_failed_metrics": false }, @@ -178,7 +176,6 @@ "evaluated_assets": null } ], - "tags": null, "notes": null, "allow_failed_metrics": false } From 4618340c94b3ad2f2e4d89b8f6e408c047bb8574 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Thu, 31 Oct 2024 22:33:47 -0700 Subject: [PATCH 12/23] chore: lint --- examples/bergamo_ophys_session.py | 2 +- examples/ephys_rig.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/bergamo_ophys_session.py b/examples/bergamo_ophys_session.py index 0e8bccd50..4af6ce9a5 100644 --- a/examples/bergamo_ophys_session.py +++ b/examples/bergamo_ophys_session.py @@ -65,7 +65,7 @@ magnification="1x", fov_scale_factor=1.5, frame_rate=20, - frame_rate_unit=FrequencyUnit.HZ + frame_rate_unit=FrequencyUnit.HZ, ), ], ), diff --git a/examples/ephys_rig.py b/examples/ephys_rig.py index 51aa6a618..4ea6f733d 100644 --- a/examples/ephys_rig.py +++ b/examples/ephys_rig.py @@ -202,7 +202,13 @@ description="850 nm longpass filter", ) -lens = Lens(name="Camera lens", focal_length=15, focal_length_unit=SizeUnit.MM, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") +lens = Lens( + name="Camera lens", + focal_length=15, + focal_length_unit=SizeUnit.MM, + manufacturer=Organization.EDMUND_OPTICS, + max_aperture="f/2", +) face_camera = Camera( name="Face Camera", From 1b0e4c0bbd3ad6ffc63eb7d0fe1cc730854c7a4e Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 1 Nov 2024 11:22:07 -0700 Subject: [PATCH 13/23] tests: fix base test --- tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_base.py b/tests/test_base.py index 30103ae48..737e979f5 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -85,10 +85,10 @@ class MultiModel(AindModel): self.assertRaises(ValidationError, lambda: MultiModel(value_one_with_depth="value")) - test2 = MultiModel(value_one_with_depth="value1", value_unit="unit") + test2 = MultiModel(value_multi_one_with_depth="value1", value_multi_unit="unit") self.assertIsNotNone(test2) - test3 = MultiModel(value_unit="unit") + test3 = MultiModel(value_multi_unit="unit") self.assertIsNotNone(test3) def test_is_dict_corrupt(self): From c41b2787e3fa00217259bf3140b6e076f373172b Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 4 Nov 2024 20:58:56 -0800 Subject: [PATCH 14/23] tests: fix missing units --- tests/test_procedures.py | 6 ++++++ tests/test_rig.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/tests/test_procedures.py b/tests/test_procedures.py index 3734a698a..40fe96c17 100644 --- a/tests/test_procedures.py +++ b/tests/test_procedures.py @@ -5,6 +5,7 @@ from datetime import date from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import TimeUnit from pydantic import ValidationError from pydantic import __version__ as pyd_version @@ -60,6 +61,7 @@ def test_injection_material_check(self): injection_eye="Left", injection_duration=1, recovery_time=10, + recovery_time_unit=TimeUnit.M, ), ], ) @@ -82,6 +84,7 @@ def test_injection_material_check(self): injection_eye="Left", injection_duration=1, recovery_time=10, + recovery_time_unit=TimeUnit.M, ), ], ) @@ -113,6 +116,7 @@ def test_injection_material_check(self): injection_eye="Left", injection_duration=1, recovery_time=10, + recovery_time_unit=TimeUnit.M, ), ], ) @@ -157,6 +161,7 @@ def test_injection_material_check(self): injection_eye="Left", injection_duration=1, recovery_time=10, + recovery_time_unit=TimeUnit.M, ), IntraperitonealInjection( protocol_id="234", @@ -194,6 +199,7 @@ def test_injection_material_check(self): injection_angle=1, injection_volume=[1], recovery_time=10, + recovery_time_unit=TimeUnit.M, targeted_structure="VISp6", ), FiberImplant( diff --git a/tests/test_rig.py b/tests/test_rig.py index 0eed7daae..fb5e4c03e 100644 --- a/tests/test_rig.py +++ b/tests/test_rig.py @@ -6,6 +6,7 @@ from aind_data_schema_models.modalities import Modality from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import FrequencyUnit from pydantic import ValidationError from pydantic_core import PydanticSerializationError @@ -133,6 +134,7 @@ def test_constructors(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -149,6 +151,7 @@ def test_constructors(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -394,6 +397,7 @@ def test_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -410,6 +414,7 @@ def test_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -523,6 +528,7 @@ def test_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -544,6 +550,7 @@ def test_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", From 7c60ec840cf58789a136920329ea92e9e50f6249 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 4 Nov 2024 21:18:10 -0800 Subject: [PATCH 15/23] tests: fixing tests now missing required units --- tests/test_procedures.py | 8 +++++++- tests/test_rig.py | 5 ++++- tests/test_rig_session_compatibility.py | 9 ++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_procedures.py b/tests/test_procedures.py index 40fe96c17..871c542f0 100644 --- a/tests/test_procedures.py +++ b/tests/test_procedures.py @@ -5,7 +5,7 @@ from datetime import date from aind_data_schema_models.organizations import Organization -from aind_data_schema_models.units import TimeUnit +from aind_data_schema_models.units import TimeUnit, ConcentrationUnit from pydantic import ValidationError from pydantic import __version__ as pyd_version @@ -60,6 +60,7 @@ def test_injection_material_check(self): injection_volume=1, injection_eye="Left", injection_duration=1, + injection_duration_unit=TimeUnit.S, recovery_time=10, recovery_time_unit=TimeUnit.M, ), @@ -83,6 +84,7 @@ def test_injection_material_check(self): injection_volume=1, injection_eye="Left", injection_duration=1, + injection_duration_unit=TimeUnit.S, recovery_time=10, recovery_time_unit=TimeUnit.M, ), @@ -115,6 +117,7 @@ def test_injection_material_check(self): injection_volume=1, injection_eye="Left", injection_duration=1, + injection_duration_unit=TimeUnit.S, recovery_time=10, recovery_time_unit=TimeUnit.M, ), @@ -160,6 +163,7 @@ def test_injection_material_check(self): injection_volume=1, injection_eye="Left", injection_duration=1, + injection_duration_unit=TimeUnit.S, recovery_time=10, recovery_time_unit=TimeUnit.M, ), @@ -172,6 +176,7 @@ def test_injection_material_check(self): source=Organization.AI, lot_number="12345", concentration=1, + concentration_unit=ConcentrationUnit.UM, ) ], injection_volume=1, @@ -191,6 +196,7 @@ def test_injection_material_check(self): ) ], injection_duration=1, + injection_duration_unit=TimeUnit.S, injection_coordinate_ml=1, injection_coordinate_ap=1, injection_coordinate_depth=[1], diff --git a/tests/test_rig.py b/tests/test_rig.py index fb5e4c03e..e29ff88ff 100644 --- a/tests/test_rig.py +++ b/tests/test_rig.py @@ -713,6 +713,7 @@ def test_rig_id_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -728,6 +729,7 @@ def test_rig_id_validator(self): data_interface="USB", computer_name="ASDF", frame_rate=144, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1, sensor_height=1, chroma="Color", @@ -835,7 +837,8 @@ def test_serialize_modalities(self): expected_modalities = [{"name": "Extracellular electrophysiology", "abbreviation": "ecephys"}] # Case 1: Modality is a class instance rig_instance_modality = Rig.model_construct( - modalities=[Modality.ECEPHYS] # Example with a valid Modality instance + rig_id="123_EPHYS1-OPTO_20220101", + modalities={Modality.ECEPHYS} # Example with a valid Modality instance ) rig_json = rig_instance_modality.model_dump_json() rig_data = json.loads(rig_json) diff --git a/tests/test_rig_session_compatibility.py b/tests/test_rig_session_compatibility.py index 8be612c75..b6d2a7af3 100644 --- a/tests/test_rig_session_compatibility.py +++ b/tests/test_rig_session_compatibility.py @@ -8,6 +8,7 @@ from aind_data_schema_models.harp_types import HarpDeviceType from aind_data_schema_models.modalities import Modality from aind_data_schema_models.organizations import Organization +from aind_data_schema_models.units import FrequencyUnit, SizeUnit import aind_data_schema.components.devices as d import aind_data_schema.core.rig as r @@ -114,6 +115,7 @@ manufacturer=Organization.FLIR, computer_name=ephys_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -157,7 +159,7 @@ description="850 nm longpass filter", ) -lens = Lens(name="Camera lens", focal_length=15, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") +lens = Lens(name="Camera lens", focal_length=15, focal_length_unit=SizeUnit.MM, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") face_camera = Camera( name="Face Camera", @@ -166,6 +168,7 @@ manufacturer=Organization.FLIR, computer_name=behavior_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -188,6 +191,7 @@ manufacturer=Organization.FLIR, computer_name=behavior_computer, frame_rate=50, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=1080, sensor_height=570, sensor_format="1/2.9", @@ -489,6 +493,7 @@ def read_json(filepath: Path) -> dict: data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -518,6 +523,7 @@ def read_json(filepath: Path) -> dict: data_interface="USB", computer_name="W10DTJK7N0M3", frame_rate=120, + frame_rate_unit=FrequencyUnit.HZ, sensor_width=640, sensor_height=480, chroma="Color", @@ -697,6 +703,7 @@ def read_json(filepath: Path) -> dict: model="AC254-080-A-ML", name="Image focusing lens", focal_length=80, + focal_length_unit=SizeUnit.MM, size=1, ) ], From 910fc7b777c6feed2d0205babd087d6fd4b89da5 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 06:32:00 -0800 Subject: [PATCH 16/23] tests: fix more missing units --- tests/test_imaging.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_imaging.py b/tests/test_imaging.py index cd42fcdd0..c1940d714 100644 --- a/tests/test_imaging.py +++ b/tests/test_imaging.py @@ -5,7 +5,7 @@ from datetime import date, datetime, timezone from aind_data_schema_models.organizations import Organization -from aind_data_schema_models.units import PowerValue +from aind_data_schema_models.units import PowerValue, FrequencyUnit from pydantic import ValidationError from pydantic import __version__ as pyd_version @@ -225,36 +225,42 @@ def test_validators(self): channel_type="Analog Output", device_name="LAS-08308", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="5", channel_type="Analog Output", device_name="539251", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="4", channel_type="Analog Output", device_name="LAS-08309", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="2", channel_type="Analog Output", device_name="stage-x", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="0", channel_type="Analog Output", device_name="TL-1", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), DAQChannel( channel_name="6", channel_type="Analog Output", device_name="LAS-08307", sample_rate=10000, + sample_rate_unit=FrequencyUnit.HZ, ), ], ) @@ -272,6 +278,7 @@ def test_validators(self): " input_value=[DAQDevice(device_type='D... hardware_version=None)], input_type=list]\n" f" For further information visit https://errors.pydantic.dev/{PYD_VERSION}/v/value_error" ) + print(repr(e.exception)) self.assertEqual(expected_exception, repr(e.exception)) From 83fd227b996f891c2b3d2fdf2a518fd707b0082d Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 09:49:13 -0800 Subject: [PATCH 17/23] tests: fix examples issue --- examples/quality_control.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/quality_control.json b/examples/quality_control.json index 72646aab9..15dfb18a1 100644 --- a/examples/quality_control.json +++ b/examples/quality_control.json @@ -80,6 +80,7 @@ "evaluated_assets": null } ], + "tags": null, "notes": "", "allow_failed_metrics": false }, @@ -121,6 +122,7 @@ "evaluated_assets": null } ], + "tags": null, "notes": "Pass when video_1_num_frames==video_2_num_frames", "allow_failed_metrics": false }, @@ -176,6 +178,7 @@ "evaluated_assets": null } ], + "tags": null, "notes": null, "allow_failed_metrics": false } From 80e29fc51135c3943b3fe3cbd1cf81f8f53da184 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 09:51:32 -0800 Subject: [PATCH 18/23] chore: lint --- tests/test_metadata.py | 13 +++---------- tests/test_rig.py | 3 +-- tests/test_rig_session_compatibility.py | 8 +++++++- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index faf578b7f..78f5428df 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -379,8 +379,7 @@ def test_validate_rig_session_compatibility(self): ) def test_validate_old_schema_version(self): - """Tests that old schema versions are ignored during validation - """ + """Tests that old schema versions are ignored during validation""" m = Metadata.model_construct( name="name", location="location", @@ -498,16 +497,10 @@ def test_create_from_core_jsons_invalid(self, mock_warning: MagicMock): @patch("logging.warning") @patch("aind_data_schema.core.metadata.is_dict_corrupt") - def test_create_from_core_jsons_corrupt( - self, - mock_is_dict_corrupt: MagicMock, - mock_warning: MagicMock - ): + def test_create_from_core_jsons_corrupt(self, mock_is_dict_corrupt: MagicMock, mock_warning: MagicMock): """Tests metadata json creation ignores corrupt core jsons""" # mock corrupt procedures and processing - mock_is_dict_corrupt.side_effect = lambda x: ( - x == self.procedures_json or x == self.processing_json - ) + mock_is_dict_corrupt.side_effect = lambda x: (x == self.procedures_json or x == self.processing_json) core_jsons = { "subject": self.subject_json, "data_description": None, diff --git a/tests/test_rig.py b/tests/test_rig.py index e29ff88ff..faa5cccb0 100644 --- a/tests/test_rig.py +++ b/tests/test_rig.py @@ -837,8 +837,7 @@ def test_serialize_modalities(self): expected_modalities = [{"name": "Extracellular electrophysiology", "abbreviation": "ecephys"}] # Case 1: Modality is a class instance rig_instance_modality = Rig.model_construct( - rig_id="123_EPHYS1-OPTO_20220101", - modalities={Modality.ECEPHYS} # Example with a valid Modality instance + rig_id="123_EPHYS1-OPTO_20220101", modalities={Modality.ECEPHYS} # Example with a valid Modality instance ) rig_json = rig_instance_modality.model_dump_json() rig_data = json.loads(rig_json) diff --git a/tests/test_rig_session_compatibility.py b/tests/test_rig_session_compatibility.py index b6d2a7af3..b367a65ea 100644 --- a/tests/test_rig_session_compatibility.py +++ b/tests/test_rig_session_compatibility.py @@ -159,7 +159,13 @@ description="850 nm longpass filter", ) -lens = Lens(name="Camera lens", focal_length=15, focal_length_unit=SizeUnit.MM, manufacturer=Organization.EDMUND_OPTICS, max_aperture="f/2") +lens = Lens( + name="Camera lens", + focal_length=15, + focal_length_unit=SizeUnit.MM, + manufacturer=Organization.EDMUND_OPTICS, + max_aperture="f/2", +) face_camera = Camera( name="Face Camera", From 8f46eb046682825ed4b1e83f3aacc72bf4bcc777 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 09:54:59 -0800 Subject: [PATCH 19/23] refactor: no longer need validator on ResourceUsage --- src/aind_data_schema/core/processing.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/aind_data_schema/core/processing.py b/src/aind_data_schema/core/processing.py index f4abe9295..61186a0a2 100644 --- a/src/aind_data_schema/core/processing.py +++ b/src/aind_data_schema/core/processing.py @@ -43,15 +43,6 @@ class ResourceUsage(AindModel): ram_usage: Optional[List[ResourceTimestamped]] = Field(default=None, title="RAM usage") usage_unit: str = Field(default=UnitlessUnit.PERCENT, title="Usage unit") - @model_validator(mode="after") - def check_value_and_unit(cls, values): - """Ensure that all valued fields have units""" - if values.system_memory and not values.system_memory_unit: - raise ValueError("System memory unit is required if system memory is provided.") - if values.ram and not values.ram_unit: - raise ValueError("RAM unit is required if RAM is provided.") - return values - class DataProcess(AindModel): """Description of a single processing step""" From e2a82faa00bbd4e2a1eb153e1dee9bd858af1cf1 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 09:56:18 -0800 Subject: [PATCH 20/23] chore: lint --- src/aind_data_schema/core/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aind_data_schema/core/processing.py b/src/aind_data_schema/core/processing.py index 61186a0a2..b4f9439cc 100644 --- a/src/aind_data_schema/core/processing.py +++ b/src/aind_data_schema/core/processing.py @@ -5,7 +5,7 @@ from aind_data_schema_models.process_names import ProcessName from aind_data_schema_models.units import MemoryUnit, UnitlessUnit -from pydantic import Field, SkipValidation, ValidationInfo, field_validator, model_validator +from pydantic import Field, SkipValidation, ValidationInfo, field_validator from aind_data_schema.base import AindCoreModel, AindGeneric, AindGenericType, AindModel, AwareDatetimeWithDefault from aind_data_schema.components.tile import Tile From bd5d2834ff56b891bebb14aeffcefda183283c25 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 09:59:10 -0800 Subject: [PATCH 21/23] tests: typo in tests --- tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_base.py b/tests/test_base.py index 737e979f5..f89731c7b 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -83,7 +83,7 @@ class MultiModel(AindModel): value_multi_two_with_depth: Optional[str] = Field(default=None) value_multi_unit: Optional[str] = Field(default=None) - self.assertRaises(ValidationError, lambda: MultiModel(value_one_with_depth="value")) + self.assertRaises(ValidationError, lambda: MultiModel(value_multi_one_with_depth="value")) test2 = MultiModel(value_multi_one_with_depth="value1", value_multi_unit="unit") self.assertIsNotNone(test2) From 4b18ae08b19af42bec2d305431e36c2dc1595f4f Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Tue, 5 Nov 2024 11:14:27 -0800 Subject: [PATCH 22/23] fix: lost default --- src/aind_data_schema/core/quality_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aind_data_schema/core/quality_control.py b/src/aind_data_schema/core/quality_control.py index 86c42767e..a55268d7b 100644 --- a/src/aind_data_schema/core/quality_control.py +++ b/src/aind_data_schema/core/quality_control.py @@ -164,7 +164,7 @@ class QualityControl(AindCoreModel): _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/quality_control.py" describedBy: str = Field(default=_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL}) - schema_version: SkipValidation[Literal["1.1.2"]] = Field("1.1.2") + schema_version: SkipValidation[Literal["1.1.2"]] = Field(default="1.1.2") evaluations: List[QCEvaluation] = Field(..., title="Evaluations") notes: Optional[str] = Field(default=None, title="Notes") From a2c4f1a168d57fd20b146666a1395e078b009174 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 15 Nov 2024 17:14:46 -0800 Subject: [PATCH 23/23] refactor: remove continue statement --- src/aind_data_schema/base.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/aind_data_schema/base.py b/src/aind_data_schema/base.py index 24da5fd22..e9b3713a5 100644 --- a/src/aind_data_schema/base.py +++ b/src/aind_data_schema/base.py @@ -110,11 +110,8 @@ def unit_validator(cls, values): # Go through all the values again, if any value matches the variable name # and is set, then the unit needs to be set as well for variable_name, variable_value in values: - if variable_name == var_name: - if variable_value: - raise ValueError(f"Unit {unit_name} is required when {variable_name} is set.") - # if we found our variable and it was None, we can move on - continue + if variable_name == var_name and variable_value: + raise ValueError(f"Unit {unit_name} is required when {variable_name} is set.") # One more time, now looking for the multi-variable condition for variable_name, variable_value in values: