Merge branch 'master' into issue8245

e2e/keywords/longhorn.resource

@@ -82,3 +82,11 @@ Delete instance-manager of deployment ${deployment_id} volume
 
 Wait for Longhorn components all running
     wait_for_namespace_pods_running    longhorn-system
+
+Install Longhorn stable version
+    install_longhorn_system    is_stable_version=True
+
+Uninstall Longhorn stable version
+    ${backups_before_uninstall} =     list_all_backups    
+    uninstall_longhorn_system    is_stable_version=True
+    Set Test Variable    ${backups_before_uninstall}

e2e/keywords/node.resource

@@ -11,6 +11,11 @@ Add ${disk_type} type disk ${disk_path} for all worker nodes
         add_disk    ${worker_node}    ${disk_type}    ${disk_path}
     END
 
+Set node ${node_id} with
+    [Arguments]    &{config}
+    ${node_name} =    get_node_by_index    ${node_id}
+    set_node   ${node_name}    &{config}
+
 Disable node ${node_id} scheduling
     ${node_name} =    get_node_by_index    ${node_id}
     disable_node_scheduling    ${node_name}

e2e/keywords/sharemanager.resource

@@ -21,12 +21,12 @@ Check sharemanager ${condition} using headless service
 Wait for all sharemanager to be deleted
     wait_for_sharemanagers_deleted
 
-Delete sharemanager of deployment ${deployment_id} and wait for recreation
+Delete sharemanager pod of deployment ${deployment_id} and wait for recreation
     ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
     ${volume_name} =    get_workload_volume_name    ${deployment_name}
-    delete_sharemanager_and_wait_for_recreation    ${volume_name}
+    delete_sharemanager_pod_and_wait_for_recreation    ${volume_name}
 
-Wait for sharemanager of deployment ${deployment_id} running
+Wait for sharemanager pod of deployment ${deployment_id} running
     ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
     ${volume_name} =    get_workload_volume_name    ${deployment_name}
-    wait_for_share_manager_running    ${volume_name}
+    wait_for_share_manager_pod_running    ${volume_name}

e2e/keywords/volume.resource

@@ -91,6 +91,10 @@ Write data ${data_id} to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     write_volume_random_data   ${volume_name}    2048    ${data_id}
 
+Write data ${data_id} ${size} MB to volume ${volume_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    write_volume_random_data   ${volume_name}    ${size}    ${data_id}
+
 Keep writing data to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     keep_writing_data    ${volume_name}
@@ -119,6 +123,13 @@ Wait for volume ${volume_id} detached
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_detached    ${volume_name}
 
+Assert volume ${volume_id} remains detached for at least ${period} seconds
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    FOR    ${i}    IN RANGE    ${period}
+        wait_for_volume_detached    ${volume_name}
+        Sleep    1
+    END
+
 Wait for volume ${volume_id} faulted
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_faulted    ${volume_name}
@@ -297,6 +308,11 @@ Check volume ${volume_id} data is backup ${backup_id} created in another cluster
     ${backup_data} =    get_backup_data_from_backup_list    ${backups_before_uninstall}    ${backup_id}
     Should Be Equal    ${current_checksum}    ${backup_data}
 
+Create volume ${volume_id} from backup ${backup_id} in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    create_volume   ${volume_name}    fromBackup=${backup_url}
+
 Create DR volume ${volume_id} from backup ${backup_id} in another cluster
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}

e2e/libs/backup/rest.py

@@ -53,8 +53,13 @@ def get(self, backup_id, volume_name):
 
     def get_from_list(self, backup_list, backup_id):
         for backup in backup_list["items"]:
-            if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
-                return backup
+            try:
+                if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
+                    return backup
+            except KeyError as e:
+                logging(f"Missing key in backup metadata: {str(e)} for backup {backup['metadata']['name']}")
+            except Exception as e:
+                logging(f"Unexpected error accessing backup {backup['metadata']['name']}: {str(e)}")
         return None
 
     def get_by_snapshot(self, volume_name, snapshot_name):

e2e/libs/keywords/backing_image_keywords.py

@@ -23,7 +23,7 @@ def cleanup_backing_images(self):
 
     def delete_backing_image_manager(self, name):
         self.backing_image.delete_backing_image_manager(name)
-    
+
     def wait_all_backing_image_managers_running(self):
         self.backing_image.wait_all_backing_image_managers_running()
 

e2e/libs/keywords/backup_keywords.py

@@ -1,4 +1,5 @@
 from backup import Backup
+
 from utility.utility import logging
 from utility.utility import get_backupstore
 

e2e/libs/keywords/backupstore_keywords.py

@@ -1,8 +1,9 @@
 from backupstore import Nfs, S3
-from utility.utility import get_backupstore
-from utility.utility import logging
+
 import os
 
+from utility.utility import get_backupstore
+
 
 class backupstore_keywords:
 

e2e/libs/keywords/common_keywords.py

@@ -1,5 +1,6 @@
 from node import Node
 from node_exec import NodeExec
+
 from utility.utility import init_k8s_api_client
 from utility.utility import generate_name_with_suffix
 

e2e/libs/keywords/engine_image_keywords.py

@@ -1,5 +1,5 @@
 from engine_image import EngineImage
-from utility.utility import logging
+
 
 class engine_image_keywords:
 

e2e/libs/keywords/engine_keywords.py

@@ -1,5 +1,5 @@
 from engine import Engine
-from utility.utility import logging
+
 
 class engine_keywords:
 

e2e/libs/keywords/host_keywords.py

@@ -1,6 +1,7 @@
-import os
 from robot.libraries.BuiltIn import BuiltIn
 
+import os
+
 from host import Harvester, Aws
 from host.constant import NODE_REBOOT_DOWN_TIME_SECOND
 

e2e/libs/keywords/k8s_keywords.py

@@ -1,5 +1,7 @@
-import asyncio
 from robot.libraries.BuiltIn import BuiltIn
+
+import asyncio
+
 from k8s.k8s import restart_kubelet
 from k8s.k8s import delete_node
 from k8s.k8s import drain_node, force_drain_node
@@ -10,9 +12,12 @@
 from k8s.k8s import get_instance_manager_on_node
 from k8s.k8s import check_instance_manager_pdb_not_exist
 from k8s.k8s import wait_for_namespace_pods_running
-from utility.utility import logging
+
 from node import Node
 
+from utility.utility import logging
+
+
 class k8s_keywords:
 
     async def restart_kubelet(self, node_name, downtime_in_sec):

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -1,14 +1,16 @@
 from longhorn_deploy import LonghornDeploy
+
+
 class longhorn_deploy_keywords:
 
     def __init__(self):
         self.longhorn = LonghornDeploy()
 
-    def uninstall_longhorn_system(self):
-        self.longhorn.uninstall()
+    def uninstall_longhorn_system(self, is_stable_version=False):
+        self.longhorn.uninstall(is_stable_version)
 
     def check_longhorn_crd_removed(self):
         self.longhorn.check_longhorn_crd_removed()
 
-    def install_longhorn_system(self):
-        self.longhorn.install()
+    def install_longhorn_system(self, is_stable_version=False):
+        self.longhorn.install(is_stable_version)

e2e/libs/keywords/network_keywords.py

@@ -44,6 +44,6 @@ def disconnect_node_network_without_waiting_completion(self, node_name, disconne
 
     def drop_pod_egress_traffic(self, pod_name, drop_time_in_sec):
         drop_pod_egress_traffic(pod_name, drop_time_in_sec)
-    
+
     def wait_for_block_network_pod_completed(self, pod_name, status, namespace='default'):
         wait_for_pod_status(pod_name, status, namespace)

e2e/libs/keywords/node_keywords.py

@@ -1,6 +1,8 @@
 from node import Node
+
 from utility.utility import logging
 
+
 class node_keywords:
 
     def __init__(self):
@@ -32,6 +34,10 @@ def disable_default_disk(self, node_name):
     def enable_default_disk(self, node_name):
         self.node.set_default_disk_scheduling(node_name, allowScheduling=True)
 
+    def set_node(self, node_name, allowScheduling=True, evictionRequested=False):
+        logging(f"Setting node {node_name}; scheduling={allowScheduling}; evictionRequested={evictionRequested}")
+        self.node.set_node(node_name, allowScheduling, evictionRequested)
+
     def disable_node_scheduling(self, node_name):
         self.node.set_node_scheduling(node_name, allowScheduling=False)
 

e2e/libs/keywords/persistentvolume_keywords.py

@@ -1,4 +1,5 @@
 from persistentvolume import PersistentVolume
+
 from utility.utility import logging
 
 

e2e/libs/keywords/replica_keywords.py

@@ -1,5 +1,6 @@
 from replica import Replica
 
+
 class replica_keywords:
 
     def __init__(self):

e2e/libs/keywords/setting_keywords.py

@@ -1,5 +1,6 @@
 from setting import Setting
 
+
 class setting_keywords:
 
     def __init__(self):

e2e/libs/keywords/sharemanager_keywords.py

@@ -7,7 +7,7 @@
 
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
-
+from utility.utility import get_pod, delete_pod
 
 class sharemanager_keywords:
 
@@ -48,14 +48,32 @@ def wait_for_sharemanagers_deleted(self, name=[]):
 
         assert AssertionError, f"Failed to wait for all sharemanagers to be deleted"
 
-    def delete_sharemanager(self, name):
-        return self.sharemanager.delete(name)
 
-    def delete_sharemanager_and_wait_for_recreation(self, name):        
-        sharemanager = self.sharemanager.get(name)
-        last_creation_time = sharemanager["metadata"]["creationTimestamp"]        
-        self.sharemanager.delete(name)
-        self.sharemanager.wait_for_restart(name, last_creation_time)
+    def delete_sharemanager_pod_and_wait_for_recreation(self, name):
+        sharemanager_pod_name = "share-manager-" + name
+        sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+        last_creation_time = sharemanager_pod.metadata.creation_timestamp
+        delete_pod(sharemanager_pod_name, "longhorn-system")
+
+        retry_count, retry_interval = get_retry_count_and_interval()
+        for i in range(retry_count):
+            time.sleep(retry_interval)
+            sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+            if sharemanager_pod == None:
+                continue
+            creation_time = sharemanager_pod.metadata.creation_timestamp
+            if creation_time > last_creation_time:
+                return
+
+        assert False, f"sharemanager pod {sharemanager_pod_name} not recreated"
+
+
+    def wait_for_share_manager_pod_running(self, name):
+        sharemanager_pod_name = "share-manager-" + name
+        retry_count, retry_interval = get_retry_count_and_interval()
+        for i in range(retry_count):
+            sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+            if sharemanager_pod.status.phase == "Running":
+                return
 
-    def wait_for_share_manager_running(self, name):
-        return self.sharemanager.wait_for_running(name)
+        assert False, f"sharemanager pod {sharemanager_pod_name} not running"

e2e/libs/keywords/statefulset_keywords.py

@@ -15,7 +15,6 @@
 from workload.workload import get_workload_volume_name
 
 
-
 class statefulset_keywords:
 
     def __init__(self):

e2e/libs/keywords/storageclass_keywords.py

@@ -1,6 +1,7 @@
 from utility.utility import logging
 from storageclass import StorageClass
 
+
 class storageclass_keywords:
 
     def __init__(self):

e2e/libs/keywords/workload_keywords.py

@@ -29,7 +29,6 @@
 from utility.constant import ANNOT_EXPANDED_SIZE
 from utility.constant import LABEL_LONGHORN_COMPONENT
 from utility.utility import logging
-from node.node import Node
 
 from volume import Volume
 from volume.constant import MEBIBYTE

e2e/libs/longhorn_deploy/base.py

@@ -19,7 +19,7 @@ def install(self):
         return NotImplemented
 
     @abstractmethod
-    def uninstall(self, longhorn_branch=None):
+    def uninstall(self, is_stable_version=False):
         return NotImplemented
 
     def check_longhorn_crd_removed(self):
@@ -29,17 +29,27 @@ def check_longhorn_crd_removed(self):
 
     def check_longhorn_uninstall_pod_log(self):
         logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
-        assert "error" not in logs
-        assert "level=fatal" not in logs
+        assert "level=error" not in logs, f"find string 'level=error' in uninstall log {logs}"
+        assert "level=fatal" not in logs, f"find string 'level=fatal' in uninstall log {logs}"
 
-    def install_longhorn(self):
+    def install_longhorn(self, is_stable_version=False):
         current_path=os.getcwd()
         full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH)
 
+        if is_stable_version is True:
+            cmd = ['bash', '-c', f'IS_INSTALL_STABLE_VERSION=true {full_path}']
+        else:
+            cmd = ['bash', full_path]
+
         try:
-            output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT)
+            output = subprocess.check_output(cmd, timeout=LONGHORN_INSTALL_TIMEOUT)
             logging(output)
         except subprocess.CalledProcessError as e:
-            logging(f"Error: {e.stderr}")
+            logging(f"Command failed with exit code {e.returncode}")
+            logging(f"stdout: {e.output}")
+            logging(f"stderr: {e.stderr}")
+            raise
         except subprocess.TimeoutExpired as e:
             logging(f"Command timed out after {e.timeout} seconds")
+            logging(f"stdout: {e.output}")
+            raise

e2e/libs/longhorn_deploy/longhorn_deploy.py

@@ -14,11 +14,11 @@ def __init__(self):
         elif self._method == "helm":
             self.longhorn = LonghornHelmChart()
 
-    def uninstall(self):
-        return self.longhorn.uninstall()
+    def uninstall(self, is_stable_version=False):
+        return self.longhorn.uninstall(is_stable_version)
 
     def check_longhorn_crd_removed(self):
         return self.longhorn.check_longhorn_crd_removed()
 
-    def install(self):
-        return self.longhorn.install()
+    def install(self, is_stable_version=False):
+        return self.longhorn.install(is_stable_version)

e2e/libs/longhorn_deploy/longhorn_helm_chart.py

@@ -8,7 +8,7 @@
 
 class LonghornHelmChart(Base):
 
-    def uninstall(self):
+    def uninstall(self, is_stable_version=False):
         control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
         control_plane_node = control_plane_nodes[0]
 
@@ -19,5 +19,5 @@ def uninstall(self):
         k8s.delete_namespace(namespace=LONGHORN_NAMESPACE)
         k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
 
-    def install(self):
-        self.install_longhorn()
+    def install(self, is_stable_version=False):
+        self.install_longhorn(is_stable_version)