diff --git a/octoploy/__init__.py b/octoploy/__init__.py index 3f262a6..923b987 100644 --- a/octoploy/__init__.py +++ b/octoploy/__init__.py @@ -1 +1 @@ -__version__ = '1.2.1' +__version__ = '1.2.2' diff --git a/octoploy/api/Model.py b/octoploy/api/Model.py index 9ad436b..ab7c1f0 100644 --- a/octoploy/api/Model.py +++ b/octoploy/api/Model.py @@ -20,9 +20,8 @@ class ItemDescription: def __init__(self, data): self.data = data - def get_annotation(self, key: str) -> str: - item = self.data.get('metadata', {}).get('annotations', {}).get(key) - return item + def get_annotation(self, key: str) -> Optional[str]: + return self.data.get('metadata', {}).get('annotations', {}).get(key) class PodData: diff --git a/octoploy/api/Oc.py b/octoploy/api/Oc.py index d820e87..d1e2d42 100644 --- a/octoploy/api/Oc.py +++ b/octoploy/api/Oc.py @@ -96,12 +96,12 @@ def exec(self, pod_name: str, cmd: str, args: List[str]): def switch_context(self, context: str): """ Changes the configuration context - :param context: Context + :param context: Context which should be used """ raise NotImplemented @abstractmethod - def annotate(self, name: str, key: str, value: str, namespace: Optional[str] = None): + def annotate(self, name: str, key: str, value: Optional[str], namespace: Optional[str] = None): """ Add / updates the annotation at the given item :param name: Name @@ -193,7 +193,12 @@ def exec(self, pod_name: str, cmd: str, args: List[str], namespace: Optional[str def switch_context(self, context: str): raise NotImplemented('Not available for openshift') - def annotate(self, name: str, key: str, value: str, namespace: Optional[str] = None): + def annotate(self, name: str, key: str, value: Optional[str], namespace: Optional[str] = None): + if value is None: + # Remove the annotation + self._exec(['annotate', name, key + '-'], namespace=namespace) + return + self._exec(['annotate', '--overwrite=true', name, key + '=' + value], namespace=namespace) def delete(self, name: str, namespace: str): diff --git a/octoploy/deploy/K8sObjectDeployer.py b/octoploy/deploy/K8sObjectDeployer.py index d481419..45ab01c 100644 --- a/octoploy/deploy/K8sObjectDeployer.py +++ b/octoploy/deploy/K8sObjectDeployer.py @@ -48,21 +48,23 @@ def deploy_object(self, k8s_object: BaseObj): if current_object is None: self._log_create(item_path) - current_hash = None + state_hash = None + old_state_hash = None if current_object is not None: + old_state_hash = current_object.get_annotation(self.HASH_ANNOTATION) obj_state = self._state.get_state(self._app_config.get_name(), k8s_object) if obj_state is not None and obj_state.hash != '': - current_hash = obj_state.hash + state_hash = obj_state.hash else: # Fallback to old hash location - current_hash = current_object.get_annotation(self.HASH_ANNOTATION) + state_hash = old_state_hash - if current_object is not None and current_hash is None: + if current_object is not None and state_hash is None: # Item has not been deployed with octoploy, but it does already exist self.log.warning(f'{item_path} has no state annotation, assuming no change required') self._state.visit(self._app_config.get_name(), k8s_object, hash_val) return - if current_hash == hash_val: + if state_hash == hash_val: self._state.visit(self._app_config.get_name(), k8s_object, hash_val) self.log.debug(f"{item_path} hasn't changed") return @@ -74,6 +76,9 @@ def deploy_object(self, k8s_object: BaseObj): self._state.visit(self._app_config.get_name(), k8s_object, hash_val) return + if old_state_hash is not None: + # Migrate to new state format by removing the old one + self._api.annotate(k8s_object.get_fqn(), self.HASH_ANNOTATION, None, namespace=k8s_object.namespace) self._api.apply(k8s_object.as_string(), namespace=namespace) self._state.visit(self._app_config.get_name(), k8s_object, hash_val) @@ -88,14 +93,12 @@ def delete_abandoned_objects(self): abandoned = self._state.get_not_visited(self._app_config.get_name()) for item in abandoned: namespace = item.namespace - if namespace is None: - namespace = self._root_config.get_namespace_name() - item_path = item.kind + '/' + item.name - self._log_delete(item_path) + + self._log_delete(item.fqn) if self._mode.plan: continue - self._api.delete(item_path, namespace=namespace) + self._api.delete(item.fqn, namespace=namespace) self._state.remove(item) def _reload_config(self): diff --git a/octoploy/processing/DataDiff.py b/octoploy/processing/DataDiff.py new file mode 100644 index 0000000..04b2616 --- /dev/null +++ b/octoploy/processing/DataDiff.py @@ -0,0 +1,10 @@ +from typing import Dict + + +class DataDiff: + """ + Creates a diff between two objects + """ + + def diff(self, a: Dict[str, any], b: Dict[str, any]): + pass \ No newline at end of file diff --git a/octoploy/state/StateTracking.py b/octoploy/state/StateTracking.py index 251f407..67649e9 100644 --- a/octoploy/state/StateTracking.py +++ b/octoploy/state/StateTracking.py @@ -11,48 +11,55 @@ class ObjectState: + fqn: str + """ + Fully qualified name of the object in the format + kind.group/name + """ + + visited: bool + """ + Transient flag to indicate if the object has been visited by octoploy + """ + def __init__(self): - self.context = '' - self.name = '' - self.kind = '' + self.context: str = '' + self.namespace: str = '' + + self.fqn = '' + self.hash = '' - self.namespace: Optional[str] = None self.visited = False - """ - Transient flag to indicate if the object has been visited by octoploy - """ def update_from_key(self, key: str): - segments = key.split('/') - count = len(segments) - if count > 0: - self.context = segments[0] - if count > 1: - self.namespace = segments[1] - if count > 2: - self.kind = segments[2] - if count > 3: - self.name = segments[3] + segments = key.split('/', 2) + if len(segments) < 3: + raise ValueError(f'Invalid key, must be 3 segments long {key}') + + self.context = segments[0] + self.namespace = segments[1] + self.fqn = segments[2] def parse(self, data: Dict[str, str]) -> ObjectState: - self.context = data['context'] - self.namespace = data['namespace'] - self.kind = data['kind'] - self.name = data['name'] + self.context = data.get('context') + self.namespace = data.get('namespace') + self.fqn = data.get('fqn') + if self.context is None or self.namespace is None or self.fqn is None: + raise ValueError(f'Corrupt octoploy state, could not parse {data}') + self.hash = data.get('hash', '') return self def to_dict(self) -> Dict[str, str]: return { - 'name': self.name, - 'hash': self.hash, - 'kind': self.kind, 'context': self.context, 'namespace': self.namespace, + 'fqn': self.fqn, + 'hash': self.hash, } def get_key(self) -> str: - return f'{self.context}/{self.namespace}/{self.kind}/{self.name}' + return f'{self.context}/{self.namespace}/{self.fqn}' class StateTracking(Log): @@ -158,15 +165,10 @@ def remove(self, object_state: ObjectState): @staticmethod def _k8s_to_state(context_name: str, k8s_object: BaseObj) -> ObjectState: - api_version = k8s_object.api_version - kind = k8s_object.kind - name = k8s_object.name - + # At this point we always have a namespace set for the object state = ObjectState() - state.namespace = k8s_object.namespace state.context = context_name - state.api_version = api_version - state.kind = kind - state.name = name + state.namespace = k8s_object.namespace + state.fqn = k8s_object.get_fqn() state.visited = True return state diff --git a/tests/StateTrackingTest.py b/tests/StateTrackingTest.py index d1a6ce6..ea5ae62 100644 --- a/tests/StateTrackingTest.py +++ b/tests/StateTrackingTest.py @@ -33,61 +33,60 @@ def get_dummy_api(): prj_config.create_api = get_dummy_api return prj_config - def test_move(self): - state = StateTracking(None) - data = state._state - + def _create_state_data(self): + data = {} obj = ObjectState() - obj.update_from_key('a/b/c/d') + obj.update_from_key('a/b/c.d/12') + data['a'] = obj self.assertEqual('a', obj.context) self.assertEqual('b', obj.namespace) - self.assertEqual('c', obj.kind) - self.assertEqual('d', obj.name) - data['a'] = obj + self.assertEqual('c.d/12', obj.fqn) obj = ObjectState() obj.update_from_key('a/b/c') + data['b'] = obj self.assertEqual('a', obj.context) self.assertEqual('b', obj.namespace) - self.assertEqual('c', obj.kind) - obj.name = 'name' - data['b'] = obj + self.assertEqual('c', obj.fqn) obj = ObjectState() - obj.update_from_key('a/b') + obj.update_from_key('a/b/Deployment/name') + data['c'] = obj self.assertEqual('a', obj.context) self.assertEqual('b', obj.namespace) - obj.kind = 'Deployment' - obj.name = 'name' - data['c'] = obj + return data - obj = ObjectState() - obj.update_from_key('a') - self.assertEqual('a', obj.context) - obj.namespace = 'unittest' - obj.kind = 'Deployment' - obj.name = 'name' - data['d'] = obj - init_state = dict(data) + def test_move(self): + state = StateTracking(None) + data = state._state + data.update(self._create_state_data()) - state.move('a/b/c/d', '1/2/3/4') + self.assertEqual('a', data['a'].context) + self.assertEqual('b', data['a'].namespace) + self.assertEqual('c.d/12', data['a'].fqn) + + self.assertEqual('a', data['b'].context) + self.assertEqual('b', data['b'].namespace) + self.assertEqual('c', data['b'].fqn) + + self.assertEqual('a', data['c'].context) + self.assertEqual('b', data['c'].namespace) + + state.move('a/b/c.d/12', '1/2/3/4') self.assertEqual('1', data['a'].context) self.assertEqual('2', data['a'].namespace) - self.assertEqual('3', data['a'].kind) - self.assertEqual('4', data['a'].name) - state.move('1/2/3/4', 'a/b/c/d') + self.assertEqual('3/4', data['a'].fqn) + data = state._state = self._create_state_data() - state.move('a/b/c', '1/2/3') + state.move('a/b', '1/2') self.assertEqual('1', data['a'].context) self.assertEqual('2', data['a'].namespace) - self.assertEqual('3', data['a'].kind) - self.assertEqual('d', data['a'].name) + self.assertEqual('c.d/12', data['a'].fqn) self.assertEqual('1', data['b'].context) self.assertEqual('2', data['b'].namespace) - self.assertEqual('3', data['b'].kind) - self.assertEqual('name', data['b'].name) - state.move('1/2/3', 'a/b/c') + self.assertEqual('c', data['b'].fqn) + data = state._state = self._create_state_data() def test_create_new(self): self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '', error=Exception('NotFound')) @@ -101,8 +100,28 @@ def test_create_new(self): self.assertEqual(['apply', '-f', '-'], state_update.args) self.assertStateEqual([{"context": "ABC", "hash": "e2e4634c5cd31a1b58da917e8b181b28", - "kind": "Deployment", - "name": "ABC", + "fqn": "Deployment/ABC", + "namespace": "oc-project", + }], state_update.stdin) + + def test_duplicate_kinds(self): + self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '', error=Exception('NotFound')) + self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '{}') + + octoploy.octoploy._run_app_deploy('app_deploy_test_duplicate_kinds', 'app', self._mode) + + self.assertEqual(4, len(self._dummy_api.commands)) + self.assertEqual(['get', 'ConfigMap/octoploy-state', '-o', 'json'], self._dummy_api.commands[0].args) + state_update = self._dummy_api.commands[-1] + self.assertEqual(['apply', '-f', '-'], state_update.args) + self.assertStateEqual([{"context": "app", + "hash": "aa859898df4ff9412857e720beeabfba", + "fqn": "ProviderConfig.kubernetes.crossplane.io/default", + "namespace": "oc-project", + }, + {"context": "app", + "hash": "8652aee0d35b000096f2a263c6e3eb77", + "fqn": "ProviderConfig.grafana.crossplane.io/default", "namespace": "oc-project", }], state_update.stdin) @@ -124,38 +143,31 @@ def test_deploy_all_then_app(self): self.assertStateEqual([ {"context": "ABC", "hash": "e2e4634c5cd31a1b58da917e8b181b28", - "kind": "Deployment", - "name": "ABC", + "fqn": "Deployment/ABC", "namespace": "oc-project"}, {"context": "entity-compare-api", "hash": "644921ab79abf165d8fb8304913ff1c7", - "kind": "Deployment", - "name": "8080", + "fqn": "Deployment/8080", "namespace": "oc-project"}, {"context": "favorite-api", "hash": "3005876d55a8c9af38a58a4227a377fc", - "kind": "Deployment", - "name": "8081", + "fqn": "Deployment/8081", "namespace": "oc-project"}, {"context": "cm-types", "hash": "1f4d778af0ea594402e656fd6139c584", - "kind": "ConfigMap", - "name": "config", + "fqn": "ConfigMap/config", "namespace": "oc-project"}, {"context": "ABC2", "hash": "d6e8e8f4b59b77117ec4ad267be8dcae", - "kind": "Secret", - "name": "secret", + "fqn": "Secret/secret", "namespace": "oc-project"}, {"context": "ABC2", "hash": "178859f1aa21598384610f352d314ae6", - "kind": "Secret", - "name": "plain-secret", + "fqn": "Secret/plain-secret", "namespace": "oc-project"}, {"context": "var-append", "hash": "24d921e38f585e26cdc247d8fddc260e", - "kind": "ConfigMap", - "name": "config", + "fqn": "ConfigMap/config", "namespace": "oc-project"}, ], state_update.stdin) @@ -187,38 +199,31 @@ def test_deploy_all_twice(self): self.assertEqual(['apply', '-f', '-'], state_update.args) self.assertStateEqual([{"context": "ABC", "hash": "e2e4634c5cd31a1b58da917e8b181b28", - "kind": "Deployment", - "name": "ABC", + "fqn": "Deployment/ABC", "namespace": "oc-project"}, {"context": "entity-compare-api", "hash": "644921ab79abf165d8fb8304913ff1c7", - "kind": "Deployment", - "name": "8080", + "fqn": "Deployment/8080", "namespace": "oc-project"}, {"context": "favorite-api", "hash": "3005876d55a8c9af38a58a4227a377fc", - "kind": "Deployment", - "name": "8081", + "fqn": "Deployment/8081", "namespace": "oc-project"}, {"context": "cm-types", "hash": "1f4d778af0ea594402e656fd6139c584", - "kind": "ConfigMap", - "name": "config", + "fqn": "ConfigMap/config", "namespace": "oc-project"}, {"context": "ABC2", "hash": "d6e8e8f4b59b77117ec4ad267be8dcae", - "kind": "Secret", - "name": "secret", + "fqn": "Secret/secret", "namespace": "oc-project"}, {"context": "var-append", "hash": "24d921e38f585e26cdc247d8fddc260e", - "kind": "ConfigMap", - "name": "config", + "fqn": "ConfigMap/config", "namespace": "oc-project"}, {"context": "ABC2", "hash": "178859f1aa21598384610f352d314ae6", - "kind": "Secret", - "name": "plain-secret", + "fqn": "Secret/plain-secret", "namespace": "oc-project"}, ], state_update.stdin) # Now deploy a single app @@ -245,9 +250,9 @@ def test_removed_in_repo(self): self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '''{ "apiVersion": "v1", "data": { - "state": "[{\\"apiVersion\\": \\"v1\\", \\"context\\": \\"ABC\\", \\"kind\\": \\"DeploymentConfig\\", \\"name\\": \\"ABC\\", \\"namespace\\": \\"oc-project\\"}]" + "state": "[{\\"context\\": \\"ABC\\", \\"fqn\\": \\"DeploymentConfig/ABC\\", \\"namespace\\": \\"oc-project\\"}]" }, - "kind": "ConfigMap", + "fqn": "ConfigMap", "metadata": { "name": "octoploy-state" } diff --git a/tests/app_deploy_test_duplicate_kinds/_root.yml b/tests/app_deploy_test_duplicate_kinds/_root.yml new file mode 100644 index 0000000..6e9c9a9 --- /dev/null +++ b/tests/app_deploy_test_duplicate_kinds/_root.yml @@ -0,0 +1,4 @@ +namespace: 'oc-project' +stateName: '' +vars: + globalVar: 'global' \ No newline at end of file diff --git a/tests/app_deploy_test_duplicate_kinds/app/_index.yml b/tests/app_deploy_test_duplicate_kinds/app/_index.yml new file mode 100644 index 0000000..e22ad54 --- /dev/null +++ b/tests/app_deploy_test_duplicate_kinds/app/_index.yml @@ -0,0 +1,2 @@ +enabled: true +name: 'app' \ No newline at end of file diff --git a/tests/app_deploy_test_duplicate_kinds/app/data.yml b/tests/app_deploy_test_duplicate_kinds/app/data.yml new file mode 100644 index 0000000..82234a1 --- /dev/null +++ b/tests/app_deploy_test_duplicate_kinds/app/data.yml @@ -0,0 +1,20 @@ +apiVersion: kubernetes.crossplane.io/v1alpha1 +kind: ProviderConfig +metadata: + name: default +spec: + credentials: + source: InjectedIdentity + +--- +apiVersion: grafana.crossplane.io/v1beta1 +kind: ProviderConfig +metadata: + name: default +spec: + credentials: + secretRef: + key: config + name: crossplane-grafana + namespace: crossplane-system + source: Secret \ No newline at end of file