From bf146696c6b677fe9fb97e4b2eb276a44839b3c3 Mon Sep 17 00:00:00 2001 From: gbenhaim Date: Tue, 18 Jul 2017 19:48:41 +0300 Subject: [PATCH] clod-init: Adding cloud init support Signed-off-by: gbenhaim --- lago/lago_cloud_init.py | 212 ++++++++++++++++++++ lago/paths.py | 3 + lago/plugins/vm.py | 3 + lago/prefix.py | 45 ++++- lago/templates/cloud-init-meta-data-base.j2 | 2 + lago/templates/cloud-init-user-data-base.j2 | 9 + lago/utils.py | 97 +++++++++ tests/functional-sdk/conftest.py | 6 +- tests/functional-sdk/test_cloud_init.py | 87 ++++++++ tests/functional-sdk/test_sdk_sanity.py | 4 + tests/unit/lago/test_utils.py | 120 +++++++++++ 11 files changed, 584 insertions(+), 4 deletions(-) create mode 100644 lago/lago_cloud_init.py create mode 100644 lago/templates/cloud-init-meta-data-base.j2 create mode 100644 lago/templates/cloud-init-user-data-base.j2 create mode 100644 tests/functional-sdk/test_cloud_init.py diff --git a/lago/lago_cloud_init.py b/lago/lago_cloud_init.py new file mode 100644 index 00000000..54f9eb69 --- /dev/null +++ b/lago/lago_cloud_init.py @@ -0,0 +1,212 @@ +from functools import partial +import logging +from os import path +import yaml +from textwrap import dedent +from jinja2 import Environment, PackageLoader + +import log_utils +import utils + +LOGGER = logging.getLogger(__name__) +LogTask = partial(log_utils.LogTask, logger=LOGGER) + + +def generate_from_itr( + vms, iso_dir, ssh_public_key, collect_only=False, with_threads=False +): + with LogTask('Creating cloud-init iso images'): + utils.safe_mkdir(iso_dir) + handlers = [ + partial( + generate, + vm, + iso_dir, + ssh_public_key, + collect_only, + ) for vm in vms + ] + + if with_threads: + iso_path = utils.invoke_different_funcs_in_parallel(*handlers) + else: + iso_path = [handler() for handler in handlers] + + return dict(iso_path) + + +def generate(vm, iso_dir, ssh_public_key, collect_only=False): + # Verify that the spec is not None + vm_name = vm.name() + + with LogTask('Creating cloud-init iso for {}'.format(vm_name)): + cloud_spec = vm.spec['cloud-init'] + + vm_iso_dir = path.join(iso_dir, vm_name) + utils.safe_mkdir(vm_iso_dir) + + vm_iso_path = path.join(iso_dir, '{}.iso'.format(vm_name)) + + normalized_spec = normalize_spec( + cloud_spec, + get_jinja_replacements(vm, ssh_public_key), + vm.distro(), + ) + + LOGGER.debug(normalized_spec) + + if not collect_only: + write_to_iso = [] + user_data = normalized_spec.pop('user-data') + if user_data: + user_data_dir = path.join(vm_iso_dir, 'user-data') + write_yaml_to_file( + user_data, + user_data_dir, + prefix_lines=['#cloud-config', '\n'] + ) + write_to_iso.append(user_data_dir) + + for spec_type, spec in normalized_spec.viewitems(): + out_dir = path.join(vm_iso_dir, spec_type) + write_yaml_to_file(spec, out_dir) + write_to_iso.append(out_dir) + + if write_to_iso: + gen_iso_image(vm_iso_path, write_to_iso) + else: + LOGGER.debug('{}: no specs were found'.format(vm_name)) + else: + print yaml.safe_dump(normalized_spec) + + iso_spec = vm_name, vm_iso_path + + return iso_spec + + +def get_jinja_replacements(vm, ssh_public_key): + # yapf: disable + return { + 'user-data': { + 'root_password': vm.root_password(), + 'public_key': ssh_public_key, + }, + 'meta-data': { + 'hostname': vm.name(), + }, + } +# yapf: enable + + +def normalize_spec(cloud_spec, defaults, vm_distro): + """ + For all spec type in 'jinja_replacements', load the default and user + given spec and merge them. + + Returns: + dict: the merged default and user spec + """ + normalized_spec = {} + + for spec_type, mapping in defaults.viewitems(): + normalized_spec[spec_type] = utils.deep_update( + load_default_spec(spec_type, vm_distro, **mapping), + load_given_spec(cloud_spec.get(spec_type, {}), spec_type) + ) + + return normalized_spec + + +def load_given_spec(given_spec, spec_type): + """ + Load spec_type given from the user. + If 'path' is in the spec, the file will be loaded from 'path', + otherwise the spec will be returned without a change. + + Args: + dict or list: which represents the spec + spec_type(dict): the type of the spec + + Returns: + dict or list: which represents the spec + """ + if not given_spec: + LOGGER.debug('{} spec is empty'.format(spec_type)) + return given_spec + + if 'path' in given_spec: + LOGGER.debug( + 'loading {} spec from {}'.format(spec_type, given_spec['path']) + ) + return load_spec_from_file(given_spec['path']) + + +def load_default_spec(spec_type, vm_distro, **kwargs): + """ + Load default spec_type template from lago.templates + and render it with jinja2 + + Args: + spec_type(dict): the type of the spec + kwargs(dict): k, v for jinja2 + + Returns: + dict or list: which represnets the spec + """ + + jinja_env = Environment(loader=PackageLoader('lago', 'templates')) + template_name = 'cloud-init-{}-{}.j2'.format(spec_type, vm_distro) + base_template_name = 'cloud-init-{}-base.j2'.format(spec_type) + template = jinja_env.select_template([template_name, base_template_name]) + + default_spec = template.render(**kwargs) + LOGGER.debug('default spec for {}:\n{}'.format(spec_type, default_spec)) + + return yaml.safe_load(default_spec) + + +def load_spec_from_file(path_to_file): + try: + with open(path_to_file, mode='rt') as f: + return yaml.safe_load(f) + except yaml.YAMLError: + raise LagoCloudInitParseError(path_to_file) + + +def write_yaml_to_file(spec, out_dir, prefix_lines=None, suffix_lines=None): + with open(out_dir, mode='wt') as f: + if prefix_lines: + f.writelines(prefix_lines) + yaml.safe_dump(spec, f) + if suffix_lines: + f.writelines(suffix_lines) + + +def gen_iso_image(out_file_name, files): + cmd = [ + 'genisoimage', + '-output', + out_file_name, + '-volid', + 'cidata', + '-joliet', + '-rock', + ] + + cmd.extend(files) + utils.run_command_with_validation(cmd) + + +class LagoCloudInitException(utils.LagoException): + pass + + +class LagoCloudInitParseError(LagoCloudInitException): + def __init__(self, file_path): + super(LagoCloudInitParseError, self).__init__( + dedent( + """ + Failed to parse yaml file {}. + """.format(file_path) + ) + ) diff --git a/lago/paths.py b/lago/paths.py index 172bbdcd..54f68ec7 100644 --- a/lago/paths.py +++ b/lago/paths.py @@ -71,3 +71,6 @@ def prefix_lagofile(self): def scripts(self, *args): return self.prefixed('scripts', *args) + + def cloud_init(self): + return self.prefixed('cloud-init') diff --git a/lago/plugins/vm.py b/lago/plugins/vm.py index be498308..3a7c6a22 100644 --- a/lago/plugins/vm.py +++ b/lago/plugins/vm.py @@ -462,6 +462,9 @@ def groups(self): else: return groups + def in_spec(self, key): + return key in self._spec + def name(self): return str(self._spec['name']) diff --git a/lago/prefix.py b/lago/prefix.py index e51722a5..d6923379 100644 --- a/lago/prefix.py +++ b/lago/prefix.py @@ -45,6 +45,7 @@ import log_utils import build import sdk_utils +import lago_cloud_init LOGGER = logging.getLogger(__name__) LogTask = functools.partial(log_utils.LogTask, logger=LOGGER) @@ -1193,7 +1194,8 @@ def virt_conf( template_repo=None, template_store=None, do_bootstrap=True, - do_build=True + do_build=True, + do_cloud_init=True ): """ Initializes all the virt infrastructure of the prefix, creating the @@ -1242,6 +1244,9 @@ def virt_conf( if do_build: self.build(conf['domains']) + if do_cloud_init: + self.cloud_init(self._virt_env.get_vms()) + self.save() rollback.clear() @@ -1264,6 +1269,44 @@ def build(self, conf): utils.invoke_in_parallel(build.Build.build, builders) + def cloud_init(self, vms): + def _gen_iso_spec(iso_path, device, vm_name): + return { + 'type': 'file', + 'path': iso_path, + 'dev': device, + 'format': 'iso', + 'name': '{}-cloud-init'.format(vm_name) + } + + vms = { + vm.name(): vm + for vm in vms.values() if vm.in_spec('cloud-init') + } + + free_device = { + vm.name(): utils.allocate_dev(vm.disks).next() + for vm in vms.values() + } + + with open(self.paths.ssh_id_rsa_pub(), mode='rt') as f: + ssh_public_key = f.read() + + iso_specs = lago_cloud_init.generate_from_itr( + vms=vms.values(), + iso_dir=self.paths.cloud_init(), + ssh_public_key=ssh_public_key + ) + + for vm_name, iso_path in iso_specs.viewitems(): + vms[vm_name]._spec['disks'].append( + _gen_iso_spec( + iso_path=iso_path, + device=free_device[vm_name], + vm_name=vm_name + ) + ) + @sdk_utils.expose def export_vms( self, diff --git a/lago/templates/cloud-init-meta-data-base.j2 b/lago/templates/cloud-init-meta-data-base.j2 new file mode 100644 index 00000000..f11c9538 --- /dev/null +++ b/lago/templates/cloud-init-meta-data-base.j2 @@ -0,0 +1,2 @@ +instance-id: {{ hostname }}-001 +local-hostname: {{ hostname }} \ No newline at end of file diff --git a/lago/templates/cloud-init-user-data-base.j2 b/lago/templates/cloud-init-user-data-base.j2 new file mode 100644 index 00000000..11699ce5 --- /dev/null +++ b/lago/templates/cloud-init-user-data-base.j2 @@ -0,0 +1,9 @@ +#cloud-config +users: + - name: root + ssh-authorized-keys: + - {{ public_key }} +chpasswd: + list: + - root:{{ root_password }} + expire: False \ No newline at end of file diff --git a/lago/utils.py b/lago/utils.py index 193d293a..923b6efe 100644 --- a/lago/utils.py +++ b/lago/utils.py @@ -797,6 +797,103 @@ def ver_cmp(ver1, ver2): ) +def allocate_dev(disks_spec, dev_type='sd'): + """ + Get free devices of type 'dev_type' + + Args: + disks_spec(dict): list of disks + dev_type(str): version string + + Returns: + generator which yields the next free device + """ + taken_devs = set() + for disk in disks_spec: + current_dev = disk.get('dev') + if current_dev and current_dev.startswith(dev_type): + try: + taken_devs.add(current_dev[2]) + except IndexError: + pass + + r = (i for i in xrange(ord('a'), ord('z') + 1)) + + for i in r: + dev = chr(i) + if dev not in taken_devs: + yield dev_type + dev + + +def deep_update(a, b): + """ + Recursively merge dict b into dict a. + Lists will be joined. + If a and b as the same key but its value's type, + differ between the two, the value from b will be taken. + + Args: + a(dict): + b(dict): + + Returns: + dict: The updated dict + + Raises: + LagoException: If a or b are not of type dict + """ + if not ( + isinstance(a, collections.Mapping) + and isinstance(b, collections.Mapping) + ): + raise LagoException( + textwrap.dedent( + """ + Failed to run deep_update because one of the arguments + is not a dict. + First argument type: {0} + Second argument type {1} + """.format(type(a), type(b)) + ) + ) + + for k, v in b.iteritems(): + if k in a and type(a[k]) == type(v): + if isinstance(v, list): + a[k] = a[k] + v + elif isinstance(v, collections.Mapping): + a[k] = deep_update(a[k], v) + else: + a[k] = v + else: + a[k] = v + + return a + + +def safe_mkdir(name): + """ + Create a directory if it doesn't exists. + + Args: + name(str): Where to create the dir + + Raises: + LagoUserException: If 'name' exists and + """ + if os.path.exists(name) and not os.path.isdir(name): + raise LagoException( + textwrap.dedent( + """ + Failed to create directory {}. + {} exists and is not a directory. + """ + ) + ) + elif not os.path.exists(name): + os.mkdir(name) + + class LagoException(Exception): pass diff --git a/tests/functional-sdk/conftest.py b/tests/functional-sdk/conftest.py index ad87de83..5a0865a5 100644 --- a/tests/functional-sdk/conftest.py +++ b/tests/functional-sdk/conftest.py @@ -8,14 +8,14 @@ _local_config = { 'check_patch': { - 'images': ['el7.4-base'] + 'images': ['el7.4-base-1', 'el7.4-base-2'] }, 'check_merged': { 'images': [ - 'el7.4-base-1', 'el6-base', 'fc26-base', 'fc27-base', - 'ubuntu16.04-base', 'debian8-base' + 'el7.4-base-1', 'el7.4-base-2', 'el6-base', 'fc26-base', + 'fc27-base', 'ubuntu16.04-base', 'debian8-base' ] } # noqa: E123 } diff --git a/tests/functional-sdk/test_cloud_init.py b/tests/functional-sdk/test_cloud_init.py new file mode 100644 index 00000000..f3b6291d --- /dev/null +++ b/tests/functional-sdk/test_cloud_init.py @@ -0,0 +1,87 @@ +import textwrap +import pytest +from jinja2 import BaseLoader, Environment +from textwrap import dedent +# from lago import sdk + + +@pytest.fixture(scope='module') +def images(): + """ + vm_name -> {template, do_bootstrap, cloud_init_config} + """ + + el7 = dedent( + """ + cloud-init: + user-data: + write_files: + - path: /root/test + content: bla_bla_bla + """ + ) + + el7_no_bootstrap = dedent( + """ + cloud-init: + user-data: + write_files: + - path: /root/test + content: bla_bla_bla + """ + ) + + # yapf: disable + return { + 'el7': { + 'template': 'el7.4-base-2', + 'do_bootstrap': True, + 'cloud_init_config': el7 + }, + 'el7-no-bootstrap': { + 'template': 'el7.4-base-2', + 'do_bootstrap': False, + 'cloud_init_config': el7_no_bootstrap + } + } + # yapf: enable + + +@pytest.fixture(scope='module') +def init_str(images): + init_template = textwrap.dedent( + """ + domains: + {% for vm_name, config in images.viewitems() %} + {{ vm_name }}: + bootstrap: {{ config.do_bootstrap }} + {{ cloud_init_config }} + memory: 1024 + nics: + - net: net-01 + disks: + - template_name: {{ config.template }} + type: template + name: root + dev: sda + format: qcow2 + metadata: + {{ vm_name }}: {{ vm_name }} + artifacts: + - /var/log + - /etc/hosts + - /etc/resolv.conf + {% endfor %} + + nets: + net-01: + type: nat + dhcp: + start: 100 + end: 254 + management: true + dns_domain_name: lago.local + """ + ) + template = Environment(loader=BaseLoader()).from_string(init_template) + return template.render(images=images) diff --git a/tests/functional-sdk/test_sdk_sanity.py b/tests/functional-sdk/test_sdk_sanity.py index 1618a626..4aa1b9d8 100644 --- a/tests/functional-sdk/test_sdk_sanity.py +++ b/tests/functional-sdk/test_sdk_sanity.py @@ -19,6 +19,10 @@ def init_str(images): domains: {% for vm_name, template in images.viewitems() %} {{ vm_name }}: + {%- if vm_name == 'vm-el7-4-base-2' %} + cloud-init: {} + bootstrap: False + {%- endif %} memory: 1024 nics: - net: net-02 diff --git a/tests/unit/lago/test_utils.py b/tests/unit/lago/test_utils.py index 09686669..ada6243b 100644 --- a/tests/unit/lago/test_utils.py +++ b/tests/unit/lago/test_utils.py @@ -4,6 +4,8 @@ from lago import utils +import pytest + def deep_compare(original_obj, copy_obj): assert copy_obj == original_obj @@ -107,3 +109,121 @@ def test_fallback_to_yaml(self): expected = {'one': 1} loaded_conf = utils.load_virt_stream(virt_fd=bad_json) assert deep_compare(expected, loaded_conf) + + +# yapf: disable +class TestDeepUpdate(object): + @pytest.mark.parametrize( + 'a, b, expected', + [ + ( + { + 'run_cmd': [1, 2] + }, + { + 'run_cmd': [3, 4] + }, + { + 'run_cmd': [1, 2, 3, 4] + } + ), + ( + { + 'run_cmd_1': [1, 2], + 'run_cmd_2': ['a,', 'b'] + }, + { + 'run_cmd_1': [3, 4] + }, + { + 'run_cmd_1': [1, 2, 3, 4], + 'run_cmd_2': ['a,', 'b'] + } + ), + ( + { + 'run_cmd_1': { + 'aa': [1, 2], + 'bb': 100 + }, + 'run_cmd_2': { + 'a': 1, + 'b': 2 + } + }, + { + 'run_cmd_1': { + 'aa': [3, 4], + 'bb': 'hi' + }, + 'run_cmd_2': { + 'a': 10, + 'c': 3 + } + }, + { + 'run_cmd_1': { + 'aa': [1, 2, 3, 4], + 'bb': 'hi' + }, + 'run_cmd_2': { + 'a': 10, + 'b': 2, + 'c': 3 + } + } + ), + ( + {}, {}, {} + ), + ( + { + 'run_cmd_1': { + 'a': { + 'a': 1, + 'c': None + } + }, + 'run_cmd_2': [1, 2] + }, + { + 'run_cmd_2': [3, 4], + 'run_cmd_1': { + 'a': { + 'a': 'a', + 'b': 'b' + } + }, + 'run_cmd_3': 'a' + }, + { + 'run_cmd_1': { + 'a': { + 'a': 'a', + 'b': 'b', + 'c': None + } + }, + 'run_cmd_2': [1, 2, 3, 4], + 'run_cmd_3': 'a' + } + ) + ] + ) + def test_deep_update(self, a, b, expected): + result = utils.deep_update(a, b) + assert deep_compare(result, expected) + + @pytest.mark.parametrize( + 'a, b', + [ + ({}, []), + ('a', {}), + ([], []) + ] + ) + def test_deep_update_not_supported_types(self, a, b): + with pytest.raises(utils.LagoException): + utils.deep_update(a, b) + +# yapf: enable