feat: show data diff on change

octoploy/api/Oc.py → octoploy/api/Kubectl.py

@@ -4,7 +4,8 @@
 from abc import abstractmethod
 from typing import Optional, List
 
-from octoploy.api.Model import ItemDescription, PodData
+from octoploy.api.Model import PodData
+from octoploy.k8s.BaseObj import BaseObj
 from octoploy.utils.Log import Log
 
 
@@ -29,7 +30,7 @@ def get_namespaces(self) -> List[str]:
         """
 
     @abstractmethod
-    def get(self, name: str, namespace: Optional[str] = None) -> Optional[ItemDescription]:
+    def get(self, name: str, namespace: Optional[str] = None) -> Optional[BaseObj]:
         """
         Returns the given item
         :param name: Name
@@ -38,6 +39,16 @@ def get(self, name: str, namespace: Optional[str] = None) -> Optional[ItemDescri
         """
         raise NotImplemented
 
+    @abstractmethod
+    def dry_run(self, yml: str, namespace: Optional[str] = None) -> BaseObj:
+        """
+        Applies the given yml file in dry-run mode (server-side) and returns the resulting yml
+        :param yml: Yml file
+        :param namespace: Namespace
+        :return: Stdout
+        """
+        raise NotImplemented
+
     @abstractmethod
     def apply(self, yml: str, namespace: Optional[str] = None) -> str:
         """
@@ -129,15 +140,20 @@ def get_namespaces(self) -> List[str]:
     def tag(self, source: str, dest: str, namespace: Optional[str] = None):
         self._exec(['tag', source, dest], print_out=True, namespace=namespace)
 
-    def get(self, name: str, namespace: Optional[str] = None) -> Optional[ItemDescription]:
+    def get(self, name: str, namespace: Optional[str] = None) -> Optional[BaseObj]:
         try:
             json_str = self._exec(['get', name, '-o', 'json'], namespace=namespace)
         except Exception as e:
             if 'NotFound' in str(e):
                 return None
             raise
 
-        return ItemDescription(json.loads(json_str))
+        return BaseObj(json.loads(json_str))
+
+    def dry_run(self, yml: str, namespace: Optional[str] = None, server_side_dry_run: bool = False) -> BaseObj:
+        args = ['apply', '--server-side', '--dry-run=server', '-o', 'json', '-f', '-']
+        json_str = self._exec(args, stdin=yml, namespace=namespace)
+        return BaseObj(json.loads(json_str))
 
     def apply(self, yml: str, namespace: Optional[str] = None) -> str:
         return self._exec(['apply', '-f', '-'], stdin=yml, namespace=namespace)

octoploy/api/Model.py

@@ -16,14 +16,6 @@ def get_name(self) -> str:
         pass
 
 
-class ItemDescription:
-    def __init__(self, data):
-        self.data = data
-
-    def get_annotation(self, key: str) -> Optional[str]:
-        return self.data.get('metadata', {}).get('annotations', {}).get(key)
-
-
 class PodData:
     def __init__(self):
         self.name = ''

octoploy/config/Config.py

@@ -3,7 +3,7 @@
 import os
 from typing import Optional, Dict, List
 
-from octoploy.api.Oc import Oc, K8, K8sApi
+from octoploy.api.Kubectl import Oc, K8, K8sApi
 from octoploy.config.AppConfig import AppConfig
 from octoploy.config.BaseConfig import BaseConfig
 from octoploy.processing import Constants

octoploy/config/DeploymentActionConfig.py

@@ -7,7 +7,7 @@
 if TYPE_CHECKING:
     from octoploy.config.Config import AppConfig
 
-from octoploy.api.Oc import K8sApi
+from octoploy.api.Kubectl import K8sApi
 
 
 class DeploymentActionConfig(Log):

octoploy/deploy/K8sObjectDeployer.py

@@ -1,8 +1,9 @@
 from typing import List
 
-from octoploy.api.Oc import K8sApi
+from octoploy.api.Kubectl import K8sApi
 from octoploy.config.Config import RootConfig, AppConfig, RunMode
 from octoploy.k8s.BaseObj import BaseObj
+from octoploy.k8s.K8sObjectDiff import K8sObjectDiff
 from octoploy.state.StateTracking import StateTracking
 from octoploy.utils.Log import Log, ColorFormatter
 
@@ -88,6 +89,8 @@ def _deploy_object(self, k8s_object: BaseObj):
 
         if current_object is not None:
             self._log_update(item_path)
+            if self._mode.plan:
+                K8sObjectDiff(self._api).print(current_object, k8s_object)
 
         if self._mode.plan:
             return

octoploy/k8s/BaseObj.py

@@ -26,6 +26,9 @@ def refresh(self):
         self.name = self.metadata.get('name', None)
         self.namespace = self.metadata.get('namespace', None)
 
+    def get_annotation(self, key: str) -> Optional[str]:
+        return self.metadata.get('annotations', {}).get(key)
+
     def set_namespace(self, namespace: str):
         self.metadata['namespace'] = namespace
         self.namespace = namespace

octoploy/k8s/K8sObjectDiff.py

@@ -0,0 +1,105 @@
+from typing import Dict, List
+
+from octoploy.utils.YmlWriter import YmlWriter
+
+from octoploy.api.Kubectl import K8sApi
+from octoploy.k8s.BaseObj import BaseObj
+from octoploy.utils.Log import ColorFormatter
+
+
+class K8sObjectDiff:
+    """
+    Creates nice to look at diffs of two yml files.
+    This diff ignores any changes injected by k8s itself (e.g. status, timestamps, ..)
+    """
+
+    def __init__(self, k8s: K8sApi):
+        self._api = k8s
+
+    def print(self, current: BaseObj, new: BaseObj):
+        """
+        Prints a diff
+        :param current: The current object in the cluster
+        :param new: The new object
+        """
+        current_data = self._filter_injected(current.data)
+
+        # Server side dry-run to get the same format / list sorting
+        new = self._api.dry_run(YmlWriter.dump(new.data))
+        new_data = self._filter_injected(new.data)
+        self._print_diff(current_data, new_data, [])
+
+    def _print_diff(self, current_data: Dict[str, any], new_data: Dict[str, any],
+                    context: List[str]):
+        if current_data is None:
+            current_data = {}
+        if new_data is None:
+            new_data = {}
+        all_keys = set(current_data.keys())
+        all_keys.update(new_data.keys())
+        all_keys = sorted(all_keys)
+
+        for key in all_keys:
+            current_entry = current_data.get(key)
+            new_entry = new_data.get(key)
+            self._print_value_diff(current_entry, new_entry, context + [key])
+
+    def _print_value_diff(self, current_entry, new_entry, context):
+        if isinstance(current_entry, list) or isinstance(new_entry, list):
+            if current_entry is None:
+                current_entry = []
+            if new_entry is None:
+                new_entry = []
+
+            count = max(len(current_entry), len(new_entry))
+            for i in range(count):
+                current_val = None
+                new_val = None
+                if i < len(current_entry):
+                    current_val = current_entry[i]
+                if i < len(new_entry):
+                    new_val = new_entry[i]
+                self._print_value_diff(current_val, new_val, context + [f'[{i}]'])
+            return
+
+        if isinstance(current_entry, dict) or isinstance(new_entry, dict):
+            self._print_diff(current_entry, new_entry, context)
+            return
+        if current_entry == new_entry:
+            return
+
+        if current_entry is None:
+            # The entire tree will be added
+            print(ColorFormatter.colorize(
+                '+ ' + '.'.join(context) + f' = {new_entry}',
+                ColorFormatter.green))
+            return
+        if new_entry is None:
+            # The entire tree will be removed
+            print(ColorFormatter.colorize(
+                '- ' + '.'.join(context) + f' = {current_entry}',
+                ColorFormatter.red))
+            return
+
+        print(ColorFormatter.colorize(
+            '~ ' + '.'.join(context) + f' = {current_entry} -> {new_entry}',
+            ColorFormatter.yellow))
+
+    def _filter_injected(self, data: Dict[str, any]) -> Dict[str, any]:
+        """Removes injected fields"""
+        data = dict(data)  # Create a copy
+        metadata = data.get('metadata', {})
+        self._del(metadata, 'creationTimestamp')
+        self._del(metadata, 'generation')
+        self._del(metadata, 'resourceVersion')
+        self._del(metadata, 'uid')
+        self._del(metadata, 'finalizers')
+        self._del(metadata.get('annotations', {}), 'kubectl.kubernetes.io/last-applied-configuration')
+        self._del(metadata.get('annotations', {}), 'kubectl.kubernetes.io/restartedAt')
+        self._del(data, 'status')
+        return data
+
+    @staticmethod
+    def _del(data: Dict[str, any], key: str):
+        if key in data:
+            del data[key]

octoploy/state/StateTracking.py

@@ -4,7 +4,7 @@
 
 import yaml
 
-from octoploy.api.Oc import K8sApi
+from octoploy.api.Kubectl import K8sApi
 from octoploy.k8s.BaseObj import BaseObj
 from octoploy.utils.Log import Log
 from octoploy.utils.YmlWriter import YmlWriter

octoploy/utils/Yml.py

@@ -15,3 +15,7 @@ def load_docs(path: str) -> List[Dict[any, any]]:
                     continue
                 docs.append(doc)
         return docs
+
+    @classmethod
+    def load_str(cls, yml: str) -> Dict[str, any]:
+        return yaml.load(yml, Loader=yaml.FullLoader)

octoploy/utils/YmlWriter.py

@@ -18,9 +18,13 @@ def init(cls):
     @classmethod
     def dump(cls, data) -> str:
         cls.init()
-        return yaml.dump(data, sort_keys=True, default_flow_style=False, width=float("inf"))
+        return yaml.dump(data, sort_keys=True,
+                         default_flow_style=False,
+                         width=float("inf"))
 
     @classmethod
     def dump_all(cls, data, file):
         cls.init()
-        yaml.dump_all(data, file, sort_keys=True, default_flow_style=False, width=float("inf"))
+        yaml.dump_all(data, file, sort_keys=True,
+                      default_flow_style=False,
+                      width=float("inf"))

tests/K8sObjectDiffTest.py

@@ -0,0 +1,51 @@
+from unittest import TestCase
+
+from tests.TestUtils import DummyK8sApi
+
+from octoploy.k8s.BaseObj import BaseObj
+from octoploy.k8s.K8sObjectDiff import K8sObjectDiff
+
+
+class K8sObjectDiffTest(TestCase):
+
+    def test_diff(self):
+        api = DummyK8sApi()
+        a = BaseObj({
+            'kind': 'a',
+            'apiVersion': 'v1',
+            'metadata': {
+                'annotations': {
+                    'kubectl.kubernetes.io/last-applied-configuration': 'ignore',
+                },
+                'uid': '',
+                'resourceVersion': '123',
+            },
+            'spec': {
+                'field': 'value',
+                'removed': 'value',
+                'listItemAdd': ['a'],
+                'listRemoved': ['a', 'b'],
+                'listItemRemoved': ['a', 'b'],
+                'listItemChanged': ['a', 'b'],
+                'removedDict': {'a': 'hello'}
+            }
+        })
+        b = BaseObj({
+            'kind': 'a',
+            'apiVersion': 'v1',
+            'metadata': {
+                'annotations': {
+                    'newAnnotation': 'hello',
+                },
+            },
+            'spec': {
+                'field': 'otherValue',
+                'listItemAdd': ['a', 'b'],
+                'listAdded': ['a', 'b'],
+                'listItemRemoved': ['a'],
+                'listItemChanged': ['a', 'other'],
+                'addedDict': {'a': 'hello'}
+            }
+        })
+
+        K8sObjectDiff(api).print(a, b)

tests/StateTrackingTest.py

@@ -99,7 +99,7 @@ def void(ignore):
 
     def test_create_new(self):
         self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '', error=Exception('NotFound'))
-        self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '{}')
+        self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
 
         octoploy.octoploy._run_app_deploy('app_deploy_test', 'app', self._mode)
 
@@ -115,7 +115,7 @@ def test_create_new(self):
 
     def test_duplicate_kinds(self):
         self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '', error=Exception('NotFound'))
-        self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '{}')
+        self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
 
         octoploy.octoploy._run_app_deploy('app_deploy_test_duplicate_kinds', 'app', self._mode)
 
@@ -179,7 +179,7 @@ def test_deploy_all_then_app(self):
         # Now deploy a single app
         current_state = yaml.safe_load(state_update.stdin)
         self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], json.dumps(current_state))
-        self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '{}')
+        self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
         self._dummy_api.commands = []
         octoploy.octoploy._run_app_deploy('app_deploy_test', 'app', self._mode)
 
@@ -230,11 +230,11 @@ def test_deploy_all_twice(self):
         # Now deploy a single app
         current_state = yaml.safe_load(state_update.stdin)
         self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], json.dumps(current_state))
-        self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '{}')
-        self._dummy_api.respond(['get', 'Deployment/8080', '-o', 'json'], '{}')
-        self._dummy_api.respond(['get', 'Deployment/8081', '-o', 'json'], '{}')
-        self._dummy_api.respond(['get', 'ConfigMap/config', '-o', 'json'], '{}')
-        self._dummy_api.respond(['get', 'Secret/secret', '-o', 'json'], '{}')
+        self._dummy_api.respond(['get', 'Deployment/ABC', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
+        self._dummy_api.respond(['get', 'Deployment/8080', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
+        self._dummy_api.respond(['get', 'Deployment/8081', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
+        self._dummy_api.respond(['get', 'ConfigMap/config', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
+        self._dummy_api.respond(['get', 'Secret/secret', '-o', 'json'], '{"kind": "", "apiVersion": ""}')
         self._dummy_api.commands = []
         octoploy.octoploy._run_apps_deploy('app_deploy_test', self._mode)
 
@@ -249,6 +249,7 @@ def test_deploy_all_twice(self):
     def test_removed_in_repo(self):
         self._dummy_api.respond(['get', 'DeploymentConfig/ABC', '-o', 'json'], '', error=Exception('NotFound'))
         self._dummy_api.respond(['get', 'ConfigMap/octoploy-state', '-o', 'json'], '''{
+  "kind": "test",
   "apiVersion": "v1",
   "data": {
     "state": "[{\\"context\\": \\"ABC\\", \\"fqn\\": \\"DeploymentConfig/ABC\\", \\"namespace\\": \\"oc-project\\"}]"

tests/TestUtils.py

@@ -3,7 +3,11 @@
 from typing import Optional, List
 from unittest.mock import Mock
 
-from octoploy.api.Oc import Oc
+from octoploy.utils.Yml import Yml
+
+from octoploy.api.Kubectl import Oc
+from octoploy.k8s.BaseObj import BaseObj
+from octoploy.utils import YmlWriter
 
 OCTOPLOY_KEY = 'key123'
 
@@ -32,6 +36,8 @@ def __init__(self):
         self.responds = []
         self._respond_not_found = False
 
+    def dry_run(self, yml: str, namespace: Optional[str] = None, server_side_dry_run: bool = False) -> BaseObj:
+        return BaseObj(Yml.load_str(yml))
     def not_found_by_default(self):
         self._respond_not_found = True
 
@@ -50,7 +56,7 @@ def _exec(self, args, print_out: bool = False, stdin: str = None, namespace: Opt
                 return stdout
         if self._respond_not_found and args[0] == 'get':
             raise Exception('NotFound')
-        return '{}'
+        return '{"kind": "", "apiVersion": ""}'
 
 
 class TestHelper: