From 8b74c7490cbd11180352739146dd2bac8cc5da5f Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 23 Oct 2023 14:34:45 +0200 Subject: [PATCH] Add new `extra_volumes` and `extra_volume_mounts` config This is the same thing as the previous `extraPvcs`, but those only allowed one volume mount per volume. `extraPvcs` is still supported though --- CHANGELOG.md | 3 ++ pygeoapi_kubernetes_papermill/notebook.py | 49 +++++++++++++++++++++++ tests/test_kubernetes_manager.py | 2 + tests/test_notebook_processor.py | 32 +++++++++++++++ 4 files changed, 86 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd737b7..2547ddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.1.8 +* Add new `extra_volumes` and `extra_volume_mounts` config + ## 1.1.7 * Remove result-link feature diff --git a/pygeoapi_kubernetes_papermill/notebook.py b/pygeoapi_kubernetes_papermill/notebook.py index fa88ee2..42262a0 100644 --- a/pygeoapi_kubernetes_papermill/notebook.py +++ b/pygeoapi_kubernetes_papermill/notebook.py @@ -187,6 +187,8 @@ def __init__(self, processor_def: dict) -> None: self.s3: Optional[Dict[str, str]] = processor_def.get("s3") self.home_volume_claim_name: str = processor_def["home_volume_claim_name"] self.extra_pvcs: List = processor_def["extra_pvcs"] + self.extra_volumes: List = processor_def["extra_volumes"] + self.extra_volume_mounts: List = processor_def["extra_volume_mounts"] self.jupyer_base_url: str = processor_def["jupyter_base_url"] self.base_output_directory: Path = Path(processor_def["output_directory"]) self.results_in_output_dir: bool = bool( @@ -419,11 +421,20 @@ def extra_configs() -> Iterable[ExtraConfig]: if self.home_volume_claim_name: yield home_volume_config(self.home_volume_claim_name) + # DEPRECATED yield from ( extra_pvc_config(extra_pvc={**extra_pvc, "num": num}) for num, extra_pvc in enumerate(self.extra_pvcs) ) + yield from ( + extra_volume_config(extra_volume) for extra_volume in self.extra_volumes + ) + yield from ( + extra_volume_mount_config(extra_volume_mount) + for extra_volume_mount in self.extra_volume_mounts + ) + if self.s3: yield s3_config( bucket_name=self.s3["bucket_name"], @@ -652,6 +663,7 @@ def home_volume_config(home_volume_claim_name: str) -> ExtraConfig: def extra_pvc_config(extra_pvc: Dict) -> ExtraConfig: + # DEPRECATED extra_name = f"extra-{extra_pvc['num']}" return ExtraConfig( volumes=[ @@ -672,6 +684,39 @@ def extra_pvc_config(extra_pvc: Dict) -> ExtraConfig: ) +def extra_volume_config(extra_volume: Dict) -> ExtraConfig: + # stupid transformer from dict to anemic k8s model + # NOTE: kubespawner/utils.py has a fancy `get_k8s_model` + # which performs the same thing but way more thoroughly. + # Trying to avoid this complexity here for now + def construct_value(k, v): + if k == "persistentVolumeClaim": + return k8s_client.V1PersistentVolumeClaimVolumeSource(**build(v)) + else: + return v + + def build(input_dict: Dict): + return { + camel_case_to_snake_case(k): construct_value(k, v) + for k, v in input_dict.items() + } + + return ExtraConfig(volumes=[k8s_client.V1Volume(**build(extra_volume))]) + + +def extra_volume_mount_config(extra_volume_mount: Dict) -> ExtraConfig: + # stupid transformer from dict to anemic k8s model + def build(input_dict: Dict): + return { + camel_case_to_snake_case(k): v + for k, v in input_dict.items() + } + + return ExtraConfig( + volume_mounts=[k8s_client.V1VolumeMount(**build(extra_volume_mount))] + ) + + def extra_secret_mount_config(secret_name: str, num: int) -> ExtraConfig: volume_name = f"secret-{num}" return ExtraConfig( @@ -959,3 +1004,7 @@ def default_kernel(is_gpu: bool, is_edc: bool) -> Optional[str]: return "edc" else: return None + + +def camel_case_to_snake_case(s: str) -> str: + return re.sub(r"(? PapermillNotebookKubernetesProcessor: "default_image": "example", "allowed_images_regex": "", "extra_pvcs": [], + "extra_volumes": [], + "extra_volume_mounts": [], "home_volume_claim_name": "user", "image_pull_secret": "", "jupyter_base_url": "", diff --git a/tests/test_notebook_processor.py b/tests/test_notebook_processor.py index c1a3efd..c8c9b38 100644 --- a/tests/test_notebook_processor.py +++ b/tests/test_notebook_processor.py @@ -68,6 +68,8 @@ def _create_processor(def_override=None) -> PapermillNotebookKubernetesProcessor "default_image": "example", "allowed_images_regex": "", "extra_pvcs": [], + "extra_volumes": [], + "extra_volume_mounts": [], "home_volume_claim_name": "user", "image_pull_secret": "", "jupyter_base_url": "", @@ -610,3 +612,33 @@ def test_custom_output_dirname_is_added_to_command( ) assert f"{dirname}/bar.ipynb" in str(job_pod_spec.pod_spec.containers[0].command) + + +def test_extra_volumes_are_added_on_request(create_pod_kwargs): + processor = _create_processor( + { + "extra_volumes": [ + { + "name": "sharedVolume", + "persistentVolumeClaim": {"claimName": "myClaimName"}, + } + ], + "extra_volume_mounts": [ + { + "name": "sharedMount", + "mountPath": "/mnt/my", + "subPath": "eurodatacube", + } + ], + } + ) + job_pod_spec = processor.create_job_pod_spec(**create_pod_kwargs) + + assert "myClaimName" in [ + v.persistent_volume_claim.claim_name + for v in job_pod_spec.pod_spec.volumes + if v.persistent_volume_claim + ] + assert "/mnt/my" in [ + m.mount_path for m in job_pod_spec.pod_spec.containers[0].volume_mounts + ]