From 4c92082f81ab2906f4e2a0e2ebfb3b5f7f9cf1ad Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Thu, 1 Mar 2018 16:48:11 +0100 Subject: [PATCH 01/17] Add kube profile spawner Reference #135 --- kubespawner/__init__.py | 3 +- kubespawner/profile_spawner.py | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 kubespawner/profile_spawner.py diff --git a/kubespawner/__init__.py b/kubespawner/__init__.py index ac04a632..b62e457f 100644 --- a/kubespawner/__init__.py +++ b/kubespawner/__init__.py @@ -14,5 +14,6 @@ instead of the more verbose import kubespawner.spawner.KubeSpawner. """ from kubespawner.spawner import KubeSpawner +from kubespawner.profile_spawner import KubeProfileSpawner -__all__ = [KubeSpawner] +__all__ = [KubeSpawner, KubeProfileSpawner] diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py new file mode 100644 index 00000000..859e46ce --- /dev/null +++ b/kubespawner/profile_spawner.py @@ -0,0 +1,74 @@ +from traitlets import ( + Instance, Type, Tuple, List, Dict, Integer, Unicode, Float, Any +) +from jupyterhub.spawner import Spawner +from kubespawner.spawner import KubeSpawner + +class KubeProfileSpawner(KubeSpawner): + + UNDEFINED_DISPLAY_NAME = "?? undefined 'display_name' ??" + + form_template = Unicode( + """ + + """, + config = True, + help = """Template to use to construct options_form text. {input_template} is replaced with + the result of formatting input_template against each item in the profiles list.""" + ) + + input_template = Unicode(""" + """, + config = True, + help = """Template to construct {input_template} in form_template. This text will be formatted + against each item in the profiles list, in order, using the following key names: + ( display, key, type ) for the first three items in the tuple, and additionally + first = "checked" (taken from first_template) for the first item in the list, so that + the first item starts selected.""" + ) + + first_template = Unicode('selected', + config=True, + help="Text to substitute as {first} in input_template" + ) + + options_form = Unicode() + + single_user_profile_list = List( + trait = Dict(), + default_value = [], + minlen = 1, + config = True, + help = """List of profiles to offer for selection. Signature is: + List(Dict()), where each item is a dictionary that has two keys: + - 'display_name': the human readable display name + - 'kubespawner_overrride': a dictionary with overrides to apply to the KubeSpawner + settings.""" + ) + + def __init__(self, *args, **kwargs): + Spawner.__init__(self, *args, **kwargs) + + def _options_form_default(self): + temp_keys = [ + { + 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), + 'key': i, + 'first': '', + } for i, p in enumerate(self.single_user_profile_list)] + temp_keys[0]['first'] = self.first_template + text = ''.join([ self.input_template.format(**tk) for tk in temp_keys ]) + return self.form_template.format(input_template=text) + + def options_from_form(self, formdata): + # Default to first profile if somehow none is provided + selected_profile = int(formdata.get('profile',[0])[0]) + options = self.single_user_profile_list[selected_profile] + self.log.debug("Applying KubeSpawner override for profile '%s'", + options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) + kubespawner_overrride = options.get('kubespawner_overrride', {}) + for k, v in kubespawner_overrride.items(): + setattr(self, k, v) + return options \ No newline at end of file From d28fb2b425858655761e17c9078488dca760d2b7 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Sat, 3 Mar 2018 20:14:47 +0100 Subject: [PATCH 02/17] clean imports --- kubespawner/profile_spawner.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py index 859e46ce..3ea711d9 100644 --- a/kubespawner/profile_spawner.py +++ b/kubespawner/profile_spawner.py @@ -1,15 +1,13 @@ -from traitlets import ( - Instance, Type, Tuple, List, Dict, Integer, Unicode, Float, Any -) from jupyterhub.spawner import Spawner from kubespawner.spawner import KubeSpawner +from traitlets import List, Dict, Unicode class KubeProfileSpawner(KubeSpawner): UNDEFINED_DISPLAY_NAME = "?? undefined 'display_name' ??" form_template = Unicode( - """ + """ @@ -71,4 +69,4 @@ def options_from_form(self, formdata): kubespawner_overrride = options.get('kubespawner_overrride', {}) for k, v in kubespawner_overrride.items(): setattr(self, k, v) - return options \ No newline at end of file + return options From a29ad4f89ef358b7703d804963a583cd6c457868 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 5 Mar 2018 10:56:28 +0100 Subject: [PATCH 03/17] Set minlen to 0 for profile_list --- kubespawner/profile_spawner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py index 3ea711d9..b2a5ba5b 100644 --- a/kubespawner/profile_spawner.py +++ b/kubespawner/profile_spawner.py @@ -37,7 +37,7 @@ class KubeProfileSpawner(KubeSpawner): single_user_profile_list = List( trait = Dict(), default_value = [], - minlen = 1, + minlen = 0, config = True, help = """List of profiles to offer for selection. Signature is: List(Dict()), where each item is a dictionary that has two keys: From 3b338ff342b848bf83edc43a4a414a7b36fe86c8 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 5 Mar 2018 11:15:50 +0100 Subject: [PATCH 04/17] disable debug hack --- kubespawner/profile_spawner.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py index b2a5ba5b..2eaa54e3 100644 --- a/kubespawner/profile_spawner.py +++ b/kubespawner/profile_spawner.py @@ -42,12 +42,13 @@ class KubeProfileSpawner(KubeSpawner): help = """List of profiles to offer for selection. Signature is: List(Dict()), where each item is a dictionary that has two keys: - 'display_name': the human readable display name - - 'kubespawner_overrride': a dictionary with overrides to apply to the KubeSpawner + - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner settings.""" ) - def __init__(self, *args, **kwargs): - Spawner.__init__(self, *args, **kwargs) + # Only for debugs + # def __init__(self, *args, **kwargs): + # Spawner.__init__(self, *args, **kwargs) def _options_form_default(self): temp_keys = [ @@ -66,7 +67,7 @@ def options_from_form(self, formdata): options = self.single_user_profile_list[selected_profile] self.log.debug("Applying KubeSpawner override for profile '%s'", options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) - kubespawner_overrride = options.get('kubespawner_overrride', {}) - for k, v in kubespawner_overrride.items(): + kubespawner_override = options.get('kubespawner_override', {}) + for k, v in kubespawner_override.items(): setattr(self, k, v) return options From 63099aaa78c5751a4561ebd36f0a275bfcb12b4e Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Wed, 7 Mar 2018 19:47:10 +0100 Subject: [PATCH 05/17] Allow none for single_user_profile_list Signed-off-by: Gaetan Semet --- kubespawner/profile_spawner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py index 2eaa54e3..1d2ee54d 100644 --- a/kubespawner/profile_spawner.py +++ b/kubespawner/profile_spawner.py @@ -36,7 +36,7 @@ class KubeProfileSpawner(KubeSpawner): single_user_profile_list = List( trait = Dict(), - default_value = [], + default_value = None, minlen = 0, config = True, help = """List of profiles to offer for selection. Signature is: @@ -51,6 +51,8 @@ class KubeProfileSpawner(KubeSpawner): # Spawner.__init__(self, *args, **kwargs) def _options_form_default(self): + if not self.single_user_profile_list: + return temp_keys = [ { 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), @@ -62,6 +64,8 @@ def _options_form_default(self): return self.form_template.format(input_template=text) def options_from_form(self, formdata): + if not self.single_user_profile_list: + return form_data # Default to first profile if somehow none is provided selected_profile = int(formdata.get('profile',[0])[0]) options = self.single_user_profile_list[selected_profile] From fceee07fddcff4e37557899466711773af117fd8 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Sun, 11 Mar 2018 17:14:30 +0100 Subject: [PATCH 06/17] Merge KubeProfileSpawner into KubeSpawner --- kubespawner/__init__.py | 3 +- kubespawner/profile_spawner.py | 77 ---------------------------------- kubespawner/spawner.py | 68 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 79 deletions(-) delete mode 100644 kubespawner/profile_spawner.py diff --git a/kubespawner/__init__.py b/kubespawner/__init__.py index b62e457f..ac04a632 100644 --- a/kubespawner/__init__.py +++ b/kubespawner/__init__.py @@ -14,6 +14,5 @@ instead of the more verbose import kubespawner.spawner.KubeSpawner. """ from kubespawner.spawner import KubeSpawner -from kubespawner.profile_spawner import KubeProfileSpawner -__all__ = [KubeSpawner, KubeProfileSpawner] +__all__ = [KubeSpawner] diff --git a/kubespawner/profile_spawner.py b/kubespawner/profile_spawner.py deleted file mode 100644 index 1d2ee54d..00000000 --- a/kubespawner/profile_spawner.py +++ /dev/null @@ -1,77 +0,0 @@ -from jupyterhub.spawner import Spawner -from kubespawner.spawner import KubeSpawner -from traitlets import List, Dict, Unicode - -class KubeProfileSpawner(KubeSpawner): - - UNDEFINED_DISPLAY_NAME = "?? undefined 'display_name' ??" - - form_template = Unicode( - """ - - """, - config = True, - help = """Template to use to construct options_form text. {input_template} is replaced with - the result of formatting input_template against each item in the profiles list.""" - ) - - input_template = Unicode(""" - """, - config = True, - help = """Template to construct {input_template} in form_template. This text will be formatted - against each item in the profiles list, in order, using the following key names: - ( display, key, type ) for the first three items in the tuple, and additionally - first = "checked" (taken from first_template) for the first item in the list, so that - the first item starts selected.""" - ) - - first_template = Unicode('selected', - config=True, - help="Text to substitute as {first} in input_template" - ) - - options_form = Unicode() - - single_user_profile_list = List( - trait = Dict(), - default_value = None, - minlen = 0, - config = True, - help = """List of profiles to offer for selection. Signature is: - List(Dict()), where each item is a dictionary that has two keys: - - 'display_name': the human readable display name - - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner - settings.""" - ) - - # Only for debugs - # def __init__(self, *args, **kwargs): - # Spawner.__init__(self, *args, **kwargs) - - def _options_form_default(self): - if not self.single_user_profile_list: - return - temp_keys = [ - { - 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), - 'key': i, - 'first': '', - } for i, p in enumerate(self.single_user_profile_list)] - temp_keys[0]['first'] = self.first_template - text = ''.join([ self.input_template.format(**tk) for tk in temp_keys ]) - return self.form_template.format(input_template=text) - - def options_from_form(self, formdata): - if not self.single_user_profile_list: - return form_data - # Default to first profile if somehow none is provided - selected_profile = int(formdata.get('profile',[0])[0]) - options = self.single_user_profile_list[selected_profile] - self.log.debug("Applying KubeSpawner override for profile '%s'", - options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) - kubespawner_override = options.get('kubespawner_override', {}) - for k, v in kubespawner_override.items(): - setattr(self, k, v) - return options diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 4e28bfbe..afe867d0 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -57,6 +57,8 @@ class KubeSpawner(Spawner): Implement a JupyterHub spawner to spawn pods in a Kubernetes Cluster. """ + UNDEFINED_DISPLAY_NAME = "?? undefined 'display_name' ??" + # We want to have one threadpool executor that is shared across all spawner objects # This is initialized by the first spawner that is created executor = None @@ -800,6 +802,46 @@ def _hub_connect_port_default(self): """ ) + form_template = Unicode( + """ + + """, + config = True, + help = """Template to use to construct options_form text. {input_template} is replaced with + the result of formatting input_template against each item in the profiles list.""" + ) + + input_template = Unicode(""" + """, + config = True, + help = """Template to construct {input_template} in form_template. This text will be formatted + against each item in the profiles list, in order, using the following key names: + ( display, key, type ) for the first three items in the tuple, and additionally + first = "checked" (taken from first_template) for the first item in the list, so that + the first item starts selected.""" + ) + + first_template = Unicode('selected', + config=True, + help="Text to substitute as {first} in input_template" + ) + + options_form = Unicode() + + single_user_profile_list = List( + trait = Dict(), + default_value = None, + minlen = 0, + config = True, + help = """List of profiles to offer for selection. Signature is: + List(Dict()), where each item is a dictionary that has two keys: + - 'display_name': the human readable display name + - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner + settings.""" + ) + def _expand_user_properties(self, template): # Make sure username and servername match the restrictions for DNS labels safe_chars = set(string.ascii_lowercase + string.digits) @@ -1127,3 +1169,29 @@ def get_args(self): args[i] = '--hub-api-url="%s"' % (self.accessible_hub_api_url) break return args + + def _options_form_default(self): + if not self.single_user_profile_list: + return + temp_keys = [ + { + 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), + 'key': i, + 'first': '', + } for i, p in enumerate(self.single_user_profile_list)] + temp_keys[0]['first'] = self.first_template + text = ''.join([ self.input_template.format(**tk) for tk in temp_keys ]) + return self.form_template.format(input_template=text) + + def options_from_form(self, formdata): + if not self.single_user_profile_list: + return form_data + # Default to first profile if somehow none is provided + selected_profile = int(formdata.get('profile',[0])[0]) + options = self.single_user_profile_list[selected_profile] + self.log.debug("Applying KubeSpawner override for profile '%s'", + options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) + kubespawner_override = options.get('kubespawner_override', {}) + for k, v in kubespawner_override.items(): + setattr(self, k, v) + return options From b5e0aecc08a9f1361c59e34076c7f01aaf25b352 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 19 Mar 2018 13:31:20 +0100 Subject: [PATCH 07/17] rename variables after review Signed-off-by: Gaetan Semet --- kubespawner/spawner.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index afe867d0..9d15fbfd 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -803,34 +803,34 @@ def _hub_connect_port_default(self): ) form_template = Unicode( - """ + """ """, config = True, - help = """Template to use to construct options_form text. {input_template} is replaced with - the result of formatting input_template against each item in the profiles list.""" + help = """Template to use to construct options_form text. {inputs} is replaced with + the result of formatting inputs against each item in the profiles list.""" ) - input_template = Unicode(""" + inputs = Unicode(""" """, config = True, - help = """Template to construct {input_template} in form_template. This text will be formatted + help = """Template to construct {inputs} in form_template. This text will be formatted against each item in the profiles list, in order, using the following key names: ( display, key, type ) for the first three items in the tuple, and additionally - first = "checked" (taken from first_template) for the first item in the list, so that + first = "checked" (taken from first_input) for the first item in the list, so that the first item starts selected.""" ) - first_template = Unicode('selected', + first_input = Unicode('selected', config=True, - help="Text to substitute as {first} in input_template" + help="Text to substitute as {first} in inputs" ) options_form = Unicode() - single_user_profile_list = List( + profile_list = List( trait = Dict(), default_value = None, minlen = 0, @@ -1171,27 +1171,28 @@ def get_args(self): return args def _options_form_default(self): - if not self.single_user_profile_list: + if not self.profile_list: return temp_keys = [ { 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), 'key': i, 'first': '', - } for i, p in enumerate(self.single_user_profile_list)] - temp_keys[0]['first'] = self.first_template - text = ''.join([ self.input_template.format(**tk) for tk in temp_keys ]) - return self.form_template.format(input_template=text) + } for i, p in enumerate(self.profile_list)] + temp_keys[0]['first'] = self.first_input + text = ''.join([ self.inputs.format(**tk) for tk in temp_keys ]) + return self.form_template.format(inputs=text) def options_from_form(self, formdata): - if not self.single_user_profile_list: - return form_data + if not self.profile_list: + return formdata # Default to first profile if somehow none is provided selected_profile = int(formdata.get('profile',[0])[0]) - options = self.single_user_profile_list[selected_profile] + options = self.profile_list[selected_profile] self.log.debug("Applying KubeSpawner override for profile '%s'", options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) kubespawner_override = options.get('kubespawner_override', {}) for k, v in kubespawner_override.items(): + self.log.debug(".. overriding KubeSpawner value %s=%s", k, v) setattr(self, k, v) return options From 241d47b62a6f0c2cd45037202a02fc2c949e50fb Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Thu, 22 Mar 2018 13:24:29 +0100 Subject: [PATCH 08/17] Adding example in documentation Signed-off-by: Gaetan Semet --- kubespawner/spawner.py | 60 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 9d15fbfd..a15c135a 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -835,11 +835,57 @@ def _hub_connect_port_default(self): default_value = None, minlen = 0, config = True, - help = """List of profiles to offer for selection. Signature is: - List(Dict()), where each item is a dictionary that has two keys: + help = """ + List of profiles to offer for selection by the user. + + Signature is: List(Dict()), where each item is a dictionary that has two keys: - 'display_name': the human readable display name - - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner - settings.""" + - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner + settings. + + Example:: + + c.KubeSpawner.profile_list = [ + { + 'display_name': 'Training Env - Python', + 'kubespawner_override': { + 'image_spec': 'training/python:label', + 'cpu_limit': 1, + 'mem_limit': '512M', + } + }, + { + 'display_name': 'Training Env - Datascience', + 'kubespawner_override': { + 'image_spec': 'training/datascience:label', + 'cpu_limit': 4, + 'mem_limit': '8G', + } + }, { + 'display_name': 'DataScience - Small instance', + 'kubespawner_override': { + 'image_spec': 'datascience/small:label', + 'cpu_limit': 10, + 'mem_limit': '16G', + } + }, { + 'display_name': 'DataScience - Medium instance', + 'kubespawner_override': { + 'image_spec': 'datascience/medium:label', + 'cpu_limit': 48, + 'mem_limit': '96G', + } + }, { + 'display_name': 'DataScience - Medium instance (GPUx2)', + 'kubespawner_override': { + 'image_spec': 'datascience/medium:label', + 'cpu_limit': 48, + 'mem_limit': '96G', + 'extra_resource_guarantees': {"nvidia.com/gpu": "2"}, + } + } + ] + """ ) def _expand_user_properties(self, template): @@ -1175,8 +1221,8 @@ def _options_form_default(self): return temp_keys = [ { - 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), - 'key': i, + 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), + 'key': i, 'first': '', } for i, p in enumerate(self.profile_list)] temp_keys[0]['first'] = self.first_input @@ -1189,7 +1235,7 @@ def options_from_form(self, formdata): # Default to first profile if somehow none is provided selected_profile = int(formdata.get('profile',[0])[0]) options = self.profile_list[selected_profile] - self.log.debug("Applying KubeSpawner override for profile '%s'", + self.log.debug("Applying KubeSpawner override for profile '%s'", options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) kubespawner_override = options.get('kubespawner_override', {}) for k, v in kubespawner_override.items(): From ad2b4151df9e53598e53378f176c22d07e922a29 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Thu, 22 Mar 2018 13:25:28 +0100 Subject: [PATCH 09/17] trailing spaces --- kubespawner/spawner.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index a15c135a..74766252 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -632,7 +632,7 @@ def _hub_connect_port_default(self): List of access modes the user has for the pvc. The access modes are: - + - `ReadWriteOnce` – the volume can be mounted as read-write by a single node - `ReadOnlyMany` – the volume can be mounted read-only by many nodes - `ReadWriteMany` – the volume can be mounted as read-write by many nodes @@ -651,7 +651,7 @@ def _hub_connect_port_default(self): The keys is name of hooks and there are only two hooks, postStart and preStop. The values are handler of hook which executes by Kubernetes management system when hook is called. - Below is an sample copied from + Below is an sample copied from `Kubernetes doc `_ :: lifecycle: @@ -690,7 +690,7 @@ def _hub_connect_port_default(self): add: - NET_ADMIN - + See https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ for more info on what init containers are and why you might want to use them! @@ -756,7 +756,7 @@ def _hub_connect_port_default(self): which follows spec at https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#container-v1-core. One usage is setting crontab in a container to clean sensitive data with configuration below:: - + [ { 'name': 'crontab', @@ -764,7 +764,7 @@ def _hub_connect_port_default(self): 'command': ['/usr/local/bin/supercronic', '/etc/crontab'] } ] - + """ ) @@ -829,7 +829,7 @@ def _hub_connect_port_default(self): ) options_form = Unicode() - + profile_list = List( trait = Dict(), default_value = None, @@ -849,7 +849,7 @@ def _hub_connect_port_default(self): { 'display_name': 'Training Env - Python', 'kubespawner_override': { - 'image_spec': 'training/python:label', + 'singleuser_image_spec': 'training/python:label', 'cpu_limit': 1, 'mem_limit': '512M', } @@ -857,28 +857,28 @@ def _hub_connect_port_default(self): { 'display_name': 'Training Env - Datascience', 'kubespawner_override': { - 'image_spec': 'training/datascience:label', + 'singleuser_image_spec': 'training/datascience:label', 'cpu_limit': 4, 'mem_limit': '8G', } }, { 'display_name': 'DataScience - Small instance', 'kubespawner_override': { - 'image_spec': 'datascience/small:label', + 'singleuser_image_spec': 'datascience/small:label', 'cpu_limit': 10, 'mem_limit': '16G', } }, { 'display_name': 'DataScience - Medium instance', 'kubespawner_override': { - 'image_spec': 'datascience/medium:label', + 'singleuser_image_spec': 'datascience/medium:label', 'cpu_limit': 48, 'mem_limit': '96G', } }, { 'display_name': 'DataScience - Medium instance (GPUx2)', 'kubespawner_override': { - 'image_spec': 'datascience/medium:label', + 'singleuser_image_spec': 'datascience/medium:label', 'cpu_limit': 48, 'mem_limit': '96G', 'extra_resource_guarantees': {"nvidia.com/gpu": "2"}, From 7fbe23c593811233bd3135d8001578791706be50 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 26 Mar 2018 22:51:29 +0200 Subject: [PATCH 10/17] allow profile value to be callable --- kubespawner/spawner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 74766252..7739ea30 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -841,7 +841,8 @@ def _hub_connect_port_default(self): Signature is: List(Dict()), where each item is a dictionary that has two keys: - 'display_name': the human readable display name - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner - settings. + settings. Each value can be either the final value to change or a callable that + take the `KubeSpawner` instance as parameter and return the final value. Example:: @@ -1239,6 +1240,10 @@ def options_from_form(self, formdata): options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) kubespawner_override = options.get('kubespawner_override', {}) for k, v in kubespawner_override.items(): - self.log.debug(".. overriding KubeSpawner value %s=%s", k, v) + if callable(v): + v = v(self) + self.log.debug(".. overriding KubeSpawner value %s=%s (callable result)", k, v) + else: + self.log.debug(".. overriding KubeSpawner value %s=%s", k, v) setattr(self, k, v) return options From 0281b504777b0988441b77ad93ded74b31589b74 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Wed, 4 Apr 2018 19:42:37 +0200 Subject: [PATCH 11/17] add docstrings --- kubespawner/spawner.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 7739ea30..e16def70 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -57,8 +57,6 @@ class KubeSpawner(Spawner): Implement a JupyterHub spawner to spawn pods in a Kubernetes Cluster. """ - UNDEFINED_DISPLAY_NAME = "?? undefined 'display_name' ??" - # We want to have one threadpool executor that is shared across all spawner objects # This is initialized by the first spawner that is created executor = None @@ -1218,11 +1216,18 @@ def get_args(self): return args def _options_form_default(self): + ''' + Build the form template according to the `profile_list` setting. + + Returns: + None when no `profile_list` has been defined + The rendered template when `profile_list` is defined. + ''' if not self.profile_list: return temp_keys = [ { - 'display': p.get('display_name', self.UNDEFINED_DISPLAY_NAME), + 'display': p['display_name'], 'key': i, 'first': '', } for i, p in enumerate(self.profile_list)] @@ -1231,6 +1236,28 @@ def _options_form_default(self): return self.form_template.format(inputs=text) def options_from_form(self, formdata): + """get the option selected by the user on the form + + It actually reset the settings of kubespawner to each item found in the selected profile + (`kubespawner_override`). + + Args: + formdata: user selection returned by the form + + To access to the value, you can use the `get` accessor and the name of the html element, + for example:: + + formdata.get('profile',[0]) + + to get the value of the form named "profile", as defined in `form_template`:: + + + + Returns: + the selected user option + """ + if not self.profile_list: return formdata # Default to first profile if somehow none is provided From e076b5009f94e4f80ae40d578cd67248121da343 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Wed, 4 Apr 2018 19:52:30 +0200 Subject: [PATCH 12/17] update tox to 3.5 and 3.6 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index dac8d05e..307ae3d2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ minversion = 1.6 skipsdist = True -envlist = py34 +envlist = py34,py35,py36 [flake8] max-line-length = 100 From 8d0a444e07d7b7ed41fe6cee334ee5cf175d8ac4 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Wed, 4 Apr 2018 19:51:16 +0200 Subject: [PATCH 13/17] Switch to jinja rendering instead of string formatting --- kubespawner/spawner.py | 28 ++++++++++++++++------------ setup.py | 1 + 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index e16def70..9230dc3d 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -21,6 +21,7 @@ from kubernetes.client.rest import ApiException from kubernetes import client import escapism +from jinja2 import Environment, BaseLoader from .clients import shared_client from kubespawner.traitlets import Callable @@ -803,18 +804,18 @@ def _hub_connect_port_default(self): form_template = Unicode( """ """, config = True, - help = """Template to use to construct options_form text. {inputs} is replaced with + help = """Jinja2 template to use to construct options_form text. {inputs} is replaced with the result of formatting inputs against each item in the profiles list.""" ) inputs = Unicode(""" - """, + """, config = True, - help = """Template to construct {inputs} in form_template. This text will be formatted + help = """Jinja template to construct {inputs} in form_template. This text will be formatted against each item in the profiles list, in order, using the following key names: ( display, key, type ) for the first three items in the tuple, and additionally first = "checked" (taken from first_input) for the first item in the list, so that @@ -837,7 +838,7 @@ def _hub_connect_port_default(self): List of profiles to offer for selection by the user. Signature is: List(Dict()), where each item is a dictionary that has two keys: - - 'display_name': the human readable display name + - 'display_name': the human readable display name (should be HTML safe) - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner settings. Each value can be either the final value to change or a callable that take the `KubeSpawner` instance as parameter and return the final value. @@ -852,8 +853,7 @@ def _hub_connect_port_default(self): 'cpu_limit': 1, 'mem_limit': '512M', } - }, - { + }, { 'display_name': 'Training Env - Datascience', 'kubespawner_override': { 'singleuser_image_spec': 'training/datascience:label', @@ -1221,7 +1221,7 @@ def _options_form_default(self): Returns: None when no `profile_list` has been defined - The rendered template when `profile_list` is defined. + The rendered template (using jinja2) when `profile_list` is defined. ''' if not self.profile_list: return @@ -1232,8 +1232,13 @@ def _options_form_default(self): 'first': '', } for i, p in enumerate(self.profile_list)] temp_keys[0]['first'] = self.first_input - text = ''.join([ self.inputs.format(**tk) for tk in temp_keys ]) - return self.form_template.format(inputs=text) + inputs_tpl = Environment(loader=BaseLoader).from_string(self.inputs) + text = ''.join([ inputs_tpl.render(**tk) for tk in temp_keys ]) + rtemplate = Environment(loader=BaseLoader).from_string(self.form_template) + data = { + 'inputs': text, + } + return rtemplate.render(**data) def options_from_form(self, formdata): """get the option selected by the user on the form @@ -1263,8 +1268,7 @@ def options_from_form(self, formdata): # Default to first profile if somehow none is provided selected_profile = int(formdata.get('profile',[0])[0]) options = self.profile_list[selected_profile] - self.log.debug("Applying KubeSpawner override for profile '%s'", - options.get('display_name', self.UNDEFINED_DISPLAY_NAME)) + self.log.debug("Applying KubeSpawner override for profile '%s'", options['display_name']) kubespawner_override = options.get('kubespawner_override', {}) for k, v in kubespawner_override.items(): if callable(v): diff --git a/setup.py b/setup.py index bfe36a24..0aa55adb 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ 'pyYAML', 'kubernetes==4.*', 'escapism', + 'jinja2', ], setup_requires=['pytest-runner'], tests_require=['pytest'], From b3ea43b7ca38c7d29fbfa1b930efa5c7f27ebc66 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 17 Apr 2018 12:37:24 -0700 Subject: [PATCH 14/17] Return '' not None as default for options_form --- kubespawner/spawner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 9230dc3d..71fae998 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -1224,7 +1224,7 @@ def _options_form_default(self): The rendered template (using jinja2) when `profile_list` is defined. ''' if not self.profile_list: - return + return '' temp_keys = [ { 'display': p['display_name'], From 8344d1a4c4eb3cf35398f050d76b2ac9fb8197d2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 17 Apr 2018 13:31:04 -0700 Subject: [PATCH 15/17] Use a single jinja2 template for rendering forms --- kubespawner/spawner.py | 46 ++++++++++-------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 71fae998..c0a6cd14 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -801,10 +801,13 @@ def _hub_connect_port_default(self): """ ) - form_template = Unicode( - """ + profile_form_template = Unicode( + """ + """, config = True, @@ -812,23 +815,6 @@ def _hub_connect_port_default(self): the result of formatting inputs against each item in the profiles list.""" ) - inputs = Unicode(""" - """, - config = True, - help = """Jinja template to construct {inputs} in form_template. This text will be formatted - against each item in the profiles list, in order, using the following key names: - ( display, key, type ) for the first three items in the tuple, and additionally - first = "checked" (taken from first_input) for the first item in the list, so that - the first item starts selected.""" - ) - - first_input = Unicode('selected', - config=True, - help="Text to substitute as {first} in inputs" - ) - - options_form = Unicode() - profile_list = List( trait = Dict(), default_value = None, @@ -842,12 +828,14 @@ def _hub_connect_port_default(self): - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner settings. Each value can be either the final value to change or a callable that take the `KubeSpawner` instance as parameter and return the final value. + - 'default': (optional Bool) True if this is the default selected option Example:: c.KubeSpawner.profile_list = [ { 'display_name': 'Training Env - Python', + 'default': True, 'kubespawner_override': { 'singleuser_image_spec': 'training/python:label', 'cpu_limit': 1, @@ -1220,25 +1208,13 @@ def _options_form_default(self): Build the form template according to the `profile_list` setting. Returns: - None when no `profile_list` has been defined + '' when no `profile_list` has been defined The rendered template (using jinja2) when `profile_list` is defined. ''' if not self.profile_list: return '' - temp_keys = [ - { - 'display': p['display_name'], - 'key': i, - 'first': '', - } for i, p in enumerate(self.profile_list)] - temp_keys[0]['first'] = self.first_input - inputs_tpl = Environment(loader=BaseLoader).from_string(self.inputs) - text = ''.join([ inputs_tpl.render(**tk) for tk in temp_keys ]) - rtemplate = Environment(loader=BaseLoader).from_string(self.form_template) - data = { - 'inputs': text, - } - return rtemplate.render(**data) + profile_form_template = Environment(loader=BaseLoader).from_string(self.profile_form_template) + return profile_form_template.render(profile_list=self.profile_list) def options_from_form(self, formdata): """get the option selected by the user on the form From 02bb801f5c34bdbc94f79ff253ed109115240e13 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 17 Apr 2018 13:44:22 -0700 Subject: [PATCH 16/17] Update traitlet docs for profile related traitlets --- kubespawner/spawner.py | 128 ++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 60 deletions(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index c0a6cd14..b2dc3ae1 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -810,69 +810,77 @@ def _hub_connect_port_default(self): {% endfor %} """, - config = True, - help = """Jinja2 template to use to construct options_form text. {inputs} is replaced with - the result of formatting inputs against each item in the profiles list.""" - ) + config=True, + help=""" + Jinja2 template for constructing profile list shown to user. + + Used when `profile_list` is set. + + The contents of `profile_list` are passed in to the template. + This should be used to construct the contents of a HTML form. When + posted, this form is expected to have an item with name `profile` and + the value the index of the profile in `profile_list`. + """ + ) profile_list = List( - trait = Dict(), - default_value = None, - minlen = 0, - config = True, - help = """ - List of profiles to offer for selection by the user. - - Signature is: List(Dict()), where each item is a dictionary that has two keys: - - 'display_name': the human readable display name (should be HTML safe) - - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner - settings. Each value can be either the final value to change or a callable that - take the `KubeSpawner` instance as parameter and return the final value. - - 'default': (optional Bool) True if this is the default selected option - - Example:: - - c.KubeSpawner.profile_list = [ - { - 'display_name': 'Training Env - Python', - 'default': True, - 'kubespawner_override': { - 'singleuser_image_spec': 'training/python:label', - 'cpu_limit': 1, - 'mem_limit': '512M', - } - }, { - 'display_name': 'Training Env - Datascience', - 'kubespawner_override': { - 'singleuser_image_spec': 'training/datascience:label', - 'cpu_limit': 4, - 'mem_limit': '8G', - } - }, { - 'display_name': 'DataScience - Small instance', - 'kubespawner_override': { - 'singleuser_image_spec': 'datascience/small:label', - 'cpu_limit': 10, - 'mem_limit': '16G', - } - }, { - 'display_name': 'DataScience - Medium instance', - 'kubespawner_override': { - 'singleuser_image_spec': 'datascience/medium:label', - 'cpu_limit': 48, - 'mem_limit': '96G', - } - }, { - 'display_name': 'DataScience - Medium instance (GPUx2)', - 'kubespawner_override': { - 'singleuser_image_spec': 'datascience/medium:label', - 'cpu_limit': 48, - 'mem_limit': '96G', - 'extra_resource_guarantees': {"nvidia.com/gpu": "2"}, - } + trait=Dict(), + default_value=None, + minlen=0, + config=True, + help=""" + List of profiles to offer for selection by the user. + + Signature is: List(Dict()), where each item is a dictionary that has two keys: + - 'display_name': the human readable display name (should be HTML safe) + - 'kubespawner_override': a dictionary with overrides to apply to the KubeSpawner + settings. Each value can be either the final value to change or a callable that + take the `KubeSpawner` instance as parameter and return the final value. + - 'default': (optional Bool) True if this is the default selected option + + Example:: + + c.KubeSpawner.profile_list = [ + { + 'display_name': 'Training Env - Python', + 'default': True, + 'kubespawner_override': { + 'singleuser_image_spec': 'training/python:label', + 'cpu_limit': 1, + 'mem_limit': '512M', + } + }, { + 'display_name': 'Training Env - Datascience', + 'kubespawner_override': { + 'singleuser_image_spec': 'training/datascience:label', + 'cpu_limit': 4, + 'mem_limit': '8G', + } + }, { + 'display_name': 'DataScience - Small instance', + 'kubespawner_override': { + 'singleuser_image_spec': 'datascience/small:label', + 'cpu_limit': 10, + 'mem_limit': '16G', } - ] - """ + }, { + 'display_name': 'DataScience - Medium instance', + 'kubespawner_override': { + 'singleuser_image_spec': 'datascience/medium:label', + 'cpu_limit': 48, + 'mem_limit': '96G', + } + }, { + 'display_name': 'DataScience - Medium instance (GPUx2)', + 'kubespawner_override': { + 'singleuser_image_spec': 'datascience/medium:label', + 'cpu_limit': 48, + 'mem_limit': '96G', + 'extra_resource_guarantees': {"nvidia.com/gpu": "2"}, + } + } + ] + """ ) def _expand_user_properties(self, template): From efb48ae0b4d0a52149f051ef46d9c339db69e794 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 17 Apr 2018 13:51:06 -0700 Subject: [PATCH 17/17] Minor style adjustment --- kubespawner/spawner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index b2dc3ae1..0945167d 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -1250,7 +1250,7 @@ def options_from_form(self, formdata): if not self.profile_list: return formdata # Default to first profile if somehow none is provided - selected_profile = int(formdata.get('profile',[0])[0]) + selected_profile = int(formdata.get('profile', [0])[0]) options = self.profile_list[selected_profile] self.log.debug("Applying KubeSpawner override for profile '%s'", options['display_name']) kubespawner_override = options.get('kubespawner_override', {})