diff --git a/e2e/keywords/longhorn.resource b/e2e/keywords/longhorn.resource index f413066528..af4475b20a 100644 --- a/e2e/keywords/longhorn.resource +++ b/e2e/keywords/longhorn.resource @@ -82,3 +82,11 @@ Delete instance-manager of deployment ${deployment_id} volume Wait for Longhorn components all running wait_for_namespace_pods_running longhorn-system + +Install Longhorn stable version + install_longhorn_system is_stable_version=True + +Uninstall Longhorn stable version + ${backups_before_uninstall} = list_all_backups + uninstall_longhorn_system is_stable_version=True + Set Test Variable ${backups_before_uninstall} diff --git a/e2e/keywords/node.resource b/e2e/keywords/node.resource index a83ce98df9..60cdcde8ae 100644 --- a/e2e/keywords/node.resource +++ b/e2e/keywords/node.resource @@ -11,6 +11,11 @@ Add ${disk_type} type disk ${disk_path} for all worker nodes add_disk ${worker_node} ${disk_type} ${disk_path} END +Set node ${node_id} with + [Arguments] &{config} + ${node_name} = get_node_by_index ${node_id} + set_node ${node_name} &{config} + Disable node ${node_id} scheduling ${node_name} = get_node_by_index ${node_id} disable_node_scheduling ${node_name} diff --git a/e2e/keywords/sharemanager.resource b/e2e/keywords/sharemanager.resource index 6fe84fda83..3e8026de2d 100644 --- a/e2e/keywords/sharemanager.resource +++ b/e2e/keywords/sharemanager.resource @@ -21,12 +21,12 @@ Check sharemanager ${condition} using headless service Wait for all sharemanager to be deleted wait_for_sharemanagers_deleted -Delete sharemanager of deployment ${deployment_id} and wait for recreation +Delete sharemanager pod of deployment ${deployment_id} and wait for recreation ${deployment_name} = generate_name_with_suffix deployment ${deployment_id} ${volume_name} = get_workload_volume_name ${deployment_name} - delete_sharemanager_and_wait_for_recreation ${volume_name} + delete_sharemanager_pod_and_wait_for_recreation ${volume_name} -Wait for sharemanager of deployment ${deployment_id} running +Wait for sharemanager pod of deployment ${deployment_id} running ${deployment_name} = generate_name_with_suffix deployment ${deployment_id} ${volume_name} = get_workload_volume_name ${deployment_name} - wait_for_share_manager_running ${volume_name} + wait_for_share_manager_pod_running ${volume_name} diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index d17034a009..c810e193e2 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -91,6 +91,10 @@ Write data ${data_id} to volume ${volume_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} write_volume_random_data ${volume_name} 2048 ${data_id} +Write data ${data_id} ${size} MB to volume ${volume_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + write_volume_random_data ${volume_name} ${size} ${data_id} + Keep writing data to volume ${volume_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} keep_writing_data ${volume_name} @@ -119,6 +123,13 @@ Wait for volume ${volume_id} detached ${volume_name} = generate_name_with_suffix volume ${volume_id} wait_for_volume_detached ${volume_name} +Assert volume ${volume_id} remains detached for at least ${period} seconds + ${volume_name} = generate_name_with_suffix volume ${volume_id} + FOR ${i} IN RANGE ${period} + wait_for_volume_detached ${volume_name} + Sleep 1 + END + Wait for volume ${volume_id} faulted ${volume_name} = generate_name_with_suffix volume ${volume_id} wait_for_volume_faulted ${volume_name} @@ -297,6 +308,11 @@ Check volume ${volume_id} data is backup ${backup_id} created in another cluster ${backup_data} = get_backup_data_from_backup_list ${backups_before_uninstall} ${backup_id} Should Be Equal ${current_checksum} ${backup_data} +Create volume ${volume_id} from backup ${backup_id} in another cluster + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${backup_url} = get_backup_url_from_backup_list ${backups_before_uninstall} ${backup_id} + create_volume ${volume_name} fromBackup=${backup_url} + Create DR volume ${volume_id} from backup ${backup_id} in another cluster ${volume_name} = generate_name_with_suffix volume ${volume_id} ${backup_url} = get_backup_url_from_backup_list ${backups_before_uninstall} ${backup_id} diff --git a/e2e/libs/backup/rest.py b/e2e/libs/backup/rest.py index 977c13d60f..7fe9fa3dfe 100644 --- a/e2e/libs/backup/rest.py +++ b/e2e/libs/backup/rest.py @@ -53,8 +53,13 @@ def get(self, backup_id, volume_name): def get_from_list(self, backup_list, backup_id): for backup in backup_list["items"]: - if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id: - return backup + try: + if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id: + return backup + except KeyError as e: + logging(f"Missing key in backup metadata: {str(e)} for backup {backup['metadata']['name']}") + except Exception as e: + logging(f"Unexpected error accessing backup {backup['metadata']['name']}: {str(e)}") return None def get_by_snapshot(self, volume_name, snapshot_name): diff --git a/e2e/libs/keywords/backing_image_keywords.py b/e2e/libs/keywords/backing_image_keywords.py index 08927ee276..67bdf32856 100644 --- a/e2e/libs/keywords/backing_image_keywords.py +++ b/e2e/libs/keywords/backing_image_keywords.py @@ -23,7 +23,7 @@ def cleanup_backing_images(self): def delete_backing_image_manager(self, name): self.backing_image.delete_backing_image_manager(name) - + def wait_all_backing_image_managers_running(self): self.backing_image.wait_all_backing_image_managers_running() diff --git a/e2e/libs/keywords/backup_keywords.py b/e2e/libs/keywords/backup_keywords.py index c1cc0a8f0e..c586f432f1 100644 --- a/e2e/libs/keywords/backup_keywords.py +++ b/e2e/libs/keywords/backup_keywords.py @@ -1,4 +1,5 @@ from backup import Backup + from utility.utility import logging from utility.utility import get_backupstore diff --git a/e2e/libs/keywords/backupstore_keywords.py b/e2e/libs/keywords/backupstore_keywords.py index 3cec22c990..4f9a731bea 100644 --- a/e2e/libs/keywords/backupstore_keywords.py +++ b/e2e/libs/keywords/backupstore_keywords.py @@ -1,8 +1,9 @@ from backupstore import Nfs, S3 -from utility.utility import get_backupstore -from utility.utility import logging + import os +from utility.utility import get_backupstore + class backupstore_keywords: diff --git a/e2e/libs/keywords/common_keywords.py b/e2e/libs/keywords/common_keywords.py index 48c524eec3..1414875590 100644 --- a/e2e/libs/keywords/common_keywords.py +++ b/e2e/libs/keywords/common_keywords.py @@ -1,5 +1,6 @@ from node import Node from node_exec import NodeExec + from utility.utility import init_k8s_api_client from utility.utility import generate_name_with_suffix diff --git a/e2e/libs/keywords/engine_image_keywords.py b/e2e/libs/keywords/engine_image_keywords.py index dd84f05b01..e8dff8b3c4 100644 --- a/e2e/libs/keywords/engine_image_keywords.py +++ b/e2e/libs/keywords/engine_image_keywords.py @@ -1,5 +1,5 @@ from engine_image import EngineImage -from utility.utility import logging + class engine_image_keywords: diff --git a/e2e/libs/keywords/engine_keywords.py b/e2e/libs/keywords/engine_keywords.py index ebdd2425c5..c9eea58ebf 100644 --- a/e2e/libs/keywords/engine_keywords.py +++ b/e2e/libs/keywords/engine_keywords.py @@ -1,5 +1,5 @@ from engine import Engine -from utility.utility import logging + class engine_keywords: diff --git a/e2e/libs/keywords/host_keywords.py b/e2e/libs/keywords/host_keywords.py index b2a4fe64c2..99d6cc4a67 100644 --- a/e2e/libs/keywords/host_keywords.py +++ b/e2e/libs/keywords/host_keywords.py @@ -1,6 +1,7 @@ -import os from robot.libraries.BuiltIn import BuiltIn +import os + from host import Harvester, Aws from host.constant import NODE_REBOOT_DOWN_TIME_SECOND diff --git a/e2e/libs/keywords/k8s_keywords.py b/e2e/libs/keywords/k8s_keywords.py index 781abc4523..ad5f66b65f 100644 --- a/e2e/libs/keywords/k8s_keywords.py +++ b/e2e/libs/keywords/k8s_keywords.py @@ -1,5 +1,7 @@ -import asyncio from robot.libraries.BuiltIn import BuiltIn + +import asyncio + from k8s.k8s import restart_kubelet from k8s.k8s import delete_node from k8s.k8s import drain_node, force_drain_node @@ -10,9 +12,12 @@ from k8s.k8s import get_instance_manager_on_node from k8s.k8s import check_instance_manager_pdb_not_exist from k8s.k8s import wait_for_namespace_pods_running -from utility.utility import logging + from node import Node +from utility.utility import logging + + class k8s_keywords: async def restart_kubelet(self, node_name, downtime_in_sec): diff --git a/e2e/libs/keywords/longhorn_deploy_keywords.py b/e2e/libs/keywords/longhorn_deploy_keywords.py index ac170f4d5f..bd8c596b56 100644 --- a/e2e/libs/keywords/longhorn_deploy_keywords.py +++ b/e2e/libs/keywords/longhorn_deploy_keywords.py @@ -1,14 +1,16 @@ from longhorn_deploy import LonghornDeploy + + class longhorn_deploy_keywords: def __init__(self): self.longhorn = LonghornDeploy() - def uninstall_longhorn_system(self): - self.longhorn.uninstall() + def uninstall_longhorn_system(self, is_stable_version=False): + self.longhorn.uninstall(is_stable_version) def check_longhorn_crd_removed(self): self.longhorn.check_longhorn_crd_removed() - def install_longhorn_system(self): - self.longhorn.install() + def install_longhorn_system(self, is_stable_version=False): + self.longhorn.install(is_stable_version) diff --git a/e2e/libs/keywords/network_keywords.py b/e2e/libs/keywords/network_keywords.py index 5b72b66e04..9566fb0708 100644 --- a/e2e/libs/keywords/network_keywords.py +++ b/e2e/libs/keywords/network_keywords.py @@ -44,6 +44,6 @@ def disconnect_node_network_without_waiting_completion(self, node_name, disconne def drop_pod_egress_traffic(self, pod_name, drop_time_in_sec): drop_pod_egress_traffic(pod_name, drop_time_in_sec) - + def wait_for_block_network_pod_completed(self, pod_name, status, namespace='default'): wait_for_pod_status(pod_name, status, namespace) diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py index bd53dcfdf6..832834f6c6 100644 --- a/e2e/libs/keywords/node_keywords.py +++ b/e2e/libs/keywords/node_keywords.py @@ -1,6 +1,8 @@ from node import Node + from utility.utility import logging + class node_keywords: def __init__(self): @@ -32,6 +34,10 @@ def disable_default_disk(self, node_name): def enable_default_disk(self, node_name): self.node.set_default_disk_scheduling(node_name, allowScheduling=True) + def set_node(self, node_name, allowScheduling=True, evictionRequested=False): + logging(f"Setting node {node_name}; scheduling={allowScheduling}; evictionRequested={evictionRequested}") + self.node.set_node(node_name, allowScheduling, evictionRequested) + def disable_node_scheduling(self, node_name): self.node.set_node_scheduling(node_name, allowScheduling=False) diff --git a/e2e/libs/keywords/persistentvolume_keywords.py b/e2e/libs/keywords/persistentvolume_keywords.py index 87a0ec569c..e160a281e6 100644 --- a/e2e/libs/keywords/persistentvolume_keywords.py +++ b/e2e/libs/keywords/persistentvolume_keywords.py @@ -1,4 +1,5 @@ from persistentvolume import PersistentVolume + from utility.utility import logging diff --git a/e2e/libs/keywords/replica_keywords.py b/e2e/libs/keywords/replica_keywords.py index 86454f4689..a9f0966c2f 100644 --- a/e2e/libs/keywords/replica_keywords.py +++ b/e2e/libs/keywords/replica_keywords.py @@ -1,5 +1,6 @@ from replica import Replica + class replica_keywords: def __init__(self): diff --git a/e2e/libs/keywords/setting_keywords.py b/e2e/libs/keywords/setting_keywords.py index 8ee4b551bb..a67ff8ada0 100644 --- a/e2e/libs/keywords/setting_keywords.py +++ b/e2e/libs/keywords/setting_keywords.py @@ -1,5 +1,6 @@ from setting import Setting + class setting_keywords: def __init__(self): diff --git a/e2e/libs/keywords/sharemanager_keywords.py b/e2e/libs/keywords/sharemanager_keywords.py index cafc00fbca..b541f5b26b 100644 --- a/e2e/libs/keywords/sharemanager_keywords.py +++ b/e2e/libs/keywords/sharemanager_keywords.py @@ -7,7 +7,7 @@ from utility.utility import get_retry_count_and_interval from utility.utility import logging - +from utility.utility import get_pod, delete_pod class sharemanager_keywords: @@ -48,14 +48,32 @@ def wait_for_sharemanagers_deleted(self, name=[]): assert AssertionError, f"Failed to wait for all sharemanagers to be deleted" - def delete_sharemanager(self, name): - return self.sharemanager.delete(name) - def delete_sharemanager_and_wait_for_recreation(self, name): - sharemanager = self.sharemanager.get(name) - last_creation_time = sharemanager["metadata"]["creationTimestamp"] - self.sharemanager.delete(name) - self.sharemanager.wait_for_restart(name, last_creation_time) + def delete_sharemanager_pod_and_wait_for_recreation(self, name): + sharemanager_pod_name = "share-manager-" + name + sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system") + last_creation_time = sharemanager_pod.metadata.creation_timestamp + delete_pod(sharemanager_pod_name, "longhorn-system") + + retry_count, retry_interval = get_retry_count_and_interval() + for i in range(retry_count): + time.sleep(retry_interval) + sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system") + if sharemanager_pod == None: + continue + creation_time = sharemanager_pod.metadata.creation_timestamp + if creation_time > last_creation_time: + return + + assert False, f"sharemanager pod {sharemanager_pod_name} not recreated" + + + def wait_for_share_manager_pod_running(self, name): + sharemanager_pod_name = "share-manager-" + name + retry_count, retry_interval = get_retry_count_and_interval() + for i in range(retry_count): + sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system") + if sharemanager_pod.status.phase == "Running": + return - def wait_for_share_manager_running(self, name): - return self.sharemanager.wait_for_running(name) + assert False, f"sharemanager pod {sharemanager_pod_name} not running" diff --git a/e2e/libs/keywords/statefulset_keywords.py b/e2e/libs/keywords/statefulset_keywords.py index 49b0f5f59f..4baa61ab61 100644 --- a/e2e/libs/keywords/statefulset_keywords.py +++ b/e2e/libs/keywords/statefulset_keywords.py @@ -15,7 +15,6 @@ from workload.workload import get_workload_volume_name - class statefulset_keywords: def __init__(self): diff --git a/e2e/libs/keywords/storageclass_keywords.py b/e2e/libs/keywords/storageclass_keywords.py index eb2aedcb3c..bbffeaf1f8 100644 --- a/e2e/libs/keywords/storageclass_keywords.py +++ b/e2e/libs/keywords/storageclass_keywords.py @@ -1,6 +1,7 @@ from utility.utility import logging from storageclass import StorageClass + class storageclass_keywords: def __init__(self): diff --git a/e2e/libs/keywords/workload_keywords.py b/e2e/libs/keywords/workload_keywords.py index 1a28f5f9c5..d34fb1567d 100644 --- a/e2e/libs/keywords/workload_keywords.py +++ b/e2e/libs/keywords/workload_keywords.py @@ -29,7 +29,6 @@ from utility.constant import ANNOT_EXPANDED_SIZE from utility.constant import LABEL_LONGHORN_COMPONENT from utility.utility import logging -from node.node import Node from volume import Volume from volume.constant import MEBIBYTE diff --git a/e2e/libs/longhorn_deploy/base.py b/e2e/libs/longhorn_deploy/base.py index 1ba6468cbf..e0162b45c2 100644 --- a/e2e/libs/longhorn_deploy/base.py +++ b/e2e/libs/longhorn_deploy/base.py @@ -19,7 +19,7 @@ def install(self): return NotImplemented @abstractmethod - def uninstall(self, longhorn_branch=None): + def uninstall(self, is_stable_version=False): return NotImplemented def check_longhorn_crd_removed(self): @@ -29,17 +29,27 @@ def check_longhorn_crd_removed(self): def check_longhorn_uninstall_pod_log(self): logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL) - assert "error" not in logs - assert "level=fatal" not in logs + assert "level=error" not in logs, f"find string 'level=error' in uninstall log {logs}" + assert "level=fatal" not in logs, f"find string 'level=fatal' in uninstall log {logs}" - def install_longhorn(self): + def install_longhorn(self, is_stable_version=False): current_path=os.getcwd() full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH) + if is_stable_version is True: + cmd = ['bash', '-c', f'IS_INSTALL_STABLE_VERSION=true {full_path}'] + else: + cmd = ['bash', full_path] + try: - output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT) + output = subprocess.check_output(cmd, timeout=LONGHORN_INSTALL_TIMEOUT) logging(output) except subprocess.CalledProcessError as e: - logging(f"Error: {e.stderr}") + logging(f"Command failed with exit code {e.returncode}") + logging(f"stdout: {e.output}") + logging(f"stderr: {e.stderr}") + raise except subprocess.TimeoutExpired as e: logging(f"Command timed out after {e.timeout} seconds") + logging(f"stdout: {e.output}") + raise diff --git a/e2e/libs/longhorn_deploy/longhorn_deploy.py b/e2e/libs/longhorn_deploy/longhorn_deploy.py index 2023e7d73d..47fed5fadb 100644 --- a/e2e/libs/longhorn_deploy/longhorn_deploy.py +++ b/e2e/libs/longhorn_deploy/longhorn_deploy.py @@ -14,11 +14,11 @@ def __init__(self): elif self._method == "helm": self.longhorn = LonghornHelmChart() - def uninstall(self): - return self.longhorn.uninstall() + def uninstall(self, is_stable_version=False): + return self.longhorn.uninstall(is_stable_version) def check_longhorn_crd_removed(self): return self.longhorn.check_longhorn_crd_removed() - def install(self): - return self.longhorn.install() + def install(self, is_stable_version=False): + return self.longhorn.install(is_stable_version) diff --git a/e2e/libs/longhorn_deploy/longhorn_helm_chart.py b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py index 47f3cd345e..67193bc1ee 100644 --- a/e2e/libs/longhorn_deploy/longhorn_helm_chart.py +++ b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py @@ -8,7 +8,7 @@ class LonghornHelmChart(Base): - def uninstall(self): + def uninstall(self, is_stable_version=False): control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") control_plane_node = control_plane_nodes[0] @@ -19,5 +19,5 @@ def uninstall(self): k8s.delete_namespace(namespace=LONGHORN_NAMESPACE) k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) - def install(self): - self.install_longhorn() + def install(self, is_stable_version=False): + self.install_longhorn(is_stable_version) diff --git a/e2e/libs/longhorn_deploy/longhorn_kubectl.py b/e2e/libs/longhorn_deploy/longhorn_kubectl.py index 960088ed99..d915bae2e7 100644 --- a/e2e/libs/longhorn_deploy/longhorn_kubectl.py +++ b/e2e/libs/longhorn_deploy/longhorn_kubectl.py @@ -9,8 +9,11 @@ class LonghornKubectl(Base): - def uninstall(self): - longhorn_branch = os.getenv("LONGHORN_REPO_BRANCH") + def uninstall(self, is_stable_version=False): + env_var = "LONGHORN_STABLE_VERSION" if is_stable_version else "LONGHORN_REPO_BRANCH" + longhorn_branch = os.getenv(env_var) + if not longhorn_branch: + raise ValueError(f"Required environment variable {env_var} is not set") control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") control_plane_node = control_plane_nodes[0] @@ -30,5 +33,5 @@ def uninstall(self): assert res, "delete uninstallation components failed" k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) - def install(self): - self.install_longhorn() + def install(self, is_stable_version=False): + self.install_longhorn(is_stable_version) diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index 17b9ed88a7..636706d492 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -10,7 +10,6 @@ from utility.utility import get_retry_count_and_interval from utility.utility import logging -from k8s.k8s import uncordon_node class Node: @@ -129,6 +128,28 @@ def filter_nodes(nodes, condition): elif role == "worker": return worker_nodes + def set_node(self, node_name: str, allowScheduling: bool, evictionRequested: bool) -> object: + for _ in range(self.retry_count): + try: + node = get_longhorn_client().by_id_node(node_name) + + get_longhorn_client().update( + node, + allowScheduling=allowScheduling, + evictionRequested=evictionRequested + ) + + node = get_longhorn_client().by_id_node(node_name) + assert node.allowScheduling == allowScheduling + assert node.evictionRequested == evictionRequested + return node + except Exception as e: + logging(f"Updating node {node_name} error: {e}") + + time.sleep(self.retry_interval) + + raise AssertionError(f"Updating node {node_name} failed") + def set_node_scheduling(self, node_name, allowScheduling=True, retry=False): node = get_longhorn_client().by_id_node(node_name) diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py index 90b0286136..c863c2cef8 100644 --- a/e2e/libs/volume/crd.py +++ b/e2e/libs/volume/crd.py @@ -162,8 +162,6 @@ def detach(self, volume_name, node_name): # https://github.com/longhorn/longhorn/issues/3715 if e.reason != "Not Found": Exception(f'exception for patching volumeattachments:', e) - if len(body['spec']['attachmentTickets']) == 0: - self.wait_for_volume_state(volume_name, "detached") def get(self, volume_name): return self.obj_api.get_namespaced_custom_object( diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 099d7c8efd..e69a5a9a41 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -4,6 +4,6 @@ directio==1.3 flake8 kubernetes==27.2.0 requests==2.32.3 -boto3==1.35.57 +boto3==1.35.69 pyyaml==6.0.2 minio==5.0.10 diff --git a/e2e/tests/negative/component_resilience.robot b/e2e/tests/negative/component_resilience.robot index 4c5cc50596..fa45633760 100644 --- a/e2e/tests/negative/component_resilience.robot +++ b/e2e/tests/negative/component_resilience.robot @@ -174,8 +174,8 @@ Test Longhorn dynamic provisioned RWX volume recovery And Wait until volume of deployment 0 replica rebuilding started on replica node Then Delete instance-manager of deployment 0 volume and wait for recover - When Delete sharemanager of deployment 0 and wait for recreation - And Wait for sharemanager of deployment 0 running + When Delete sharemanager pod of deployment 0 and wait for recreation + And Wait for sharemanager pod of deployment 0 running And Wait for deployment 0 pods stable And Check deployment 0 data in file data.txt is intact END diff --git a/e2e/tests/negative/network_disconnect.robot b/e2e/tests/negative/network_disconnect.robot index 7cab8c6226..3e0b786c54 100644 --- a/e2e/tests/negative/network_disconnect.robot +++ b/e2e/tests/negative/network_disconnect.robot @@ -164,4 +164,5 @@ Node Disconnect With Statefulset And Wait for volume 0 healthy And Detach volume 0 from attached node + And Wait for volume 0 detached Then Wait for volume 0 detached diff --git a/e2e/tests/negative/pull_backup_from_another_longhorn.robot b/e2e/tests/negative/pull_backup_from_another_longhorn.robot new file mode 100644 index 0000000000..819350ad68 --- /dev/null +++ b/e2e/tests/negative/pull_backup_from_another_longhorn.robot @@ -0,0 +1,98 @@ +*** Settings *** +Documentation Uninstallation Checks + +Test Tags negative + +Resource ../keywords/common.resource +Resource ../keywords/setting.resource +Resource ../keywords/volume.resource +Resource ../keywords/persistentvolume.resource +Resource ../keywords/persistentvolumeclaim.resource +Resource ../keywords/workload.resource +Resource ../keywords/backup.resource +Resource ../keywords/snapshot.resource +Resource ../keywords/backupstore.resource +Resource ../keywords/longhorn.resource +Library ../libs/keywords/setting_keywords.py + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 +${DATA_ENGINE} v1 + +*** Test Cases *** +Pull backup created by another Longhorn system + [Documentation] Pull backup created by another Longhorn system + ... 1. Install test version of Longhorn. + ... 2. Create volume, write data, and take backup. + ... 3. Uninstall Longhorn. + ... 4. Install test version of Longhorn. + ... 5. Restore the backup create in step 2 and verify the data. + ... 6. Uninstall Longhorn. + ... 7. Install previous version of Longhorn. + ... 8. Create volume, write data, and take backup. + ... 9. Uninstall Longhorn. + ... 10. Install test version of Longhorn. + ... 11. Restore the backup create in step 8 and verify the data. + ... + ... Important + ... - This test case need have set environment variable manually first if not run on Jenkins + ... - LONGHORN_INSTALL_METHOD : helm or manifest + ... - LONGHORN_REPO_BRANCH (ex:master) + ... - CUSTOM_LONGHORN_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_ENGINE_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE (if not using master-head) + ... - LONGHORN_STABLE_VERSION (ex:v1.6.3) + Given Set setting deleting-confirmation-flag to true + And Create volume 0 with dataEngine=${DATA_ENGINE} + And Attach volume 0 + And Wait for volume 0 healthy + And Write data 0 300 MB to volume 0 + When Create backup 0 for volume 0 + Then Verify backup list contains no error for volume 0 + And Verify backup list contains backup 0 of volume 0 + Then Uninstall Longhorn + And Check Longhorn CRD removed + + # Install current version then pull backup and verify data + Then Install Longhorn + And Set setting deleting-confirmation-flag to true + And Set backupstore + And Check backup synced from backupstore + And Create volume 1 from backup 0 in another cluster + And Wait for volume 1 detached + And Attach volume 1 + And Wait for volume 1 healthy + Then Check volume 1 data is backup 0 created in another cluster + Then Uninstall Longhorn + And Check Longhorn CRD removed + + # Install previous version and create backup + Then Install Longhorn stable version + And Set setting deleting-confirmation-flag to true + And Set backupstore + And Create volume 2 with dataEngine=${DATA_ENGINE} + And Attach volume 2 + And Wait for volume 2 healthy + And Write data 1 300 MB to volume 2 + When Create backup 1 for volume 2 + Then Verify backup list contains no error for volume 2 + And Verify backup list contains backup 1 of volume 2 + Then Uninstall Longhorn stable version + And Check Longhorn CRD removed + + # Install current version then pull backup and verify data + Then Install Longhorn + And Set backupstore + And Check backup synced from backupstore + And Create volume 3 from backup 1 in another cluster + And Wait for volume 3 detached + And Attach volume 3 + And Wait for volume 3 healthy + Then Check volume 3 data is backup 1 created in another cluster diff --git a/e2e/tests/negative/stress_cpu.robot b/e2e/tests/negative/stress_cpu.robot index 1edcb15e60..b9d0a65836 100644 --- a/e2e/tests/negative/stress_cpu.robot +++ b/e2e/tests/negative/stress_cpu.robot @@ -41,6 +41,7 @@ Stress Volume Node CPU When Volume Is Detaching and Attaching FOR ${i} IN RANGE ${LOOP_COUNT} When Stress CPU of node with volume 0 And Detach volume 0 + And Wait for volume 0 detached And Attach volume 0 And Wait for volume 0 healthy Then Check volume 0 data is intact diff --git a/e2e/tests/negative/stress_filesystem.robot b/e2e/tests/negative/stress_filesystem.robot index e67b7b8a32..85ec54de9f 100644 --- a/e2e/tests/negative/stress_filesystem.robot +++ b/e2e/tests/negative/stress_filesystem.robot @@ -42,6 +42,7 @@ Stress Volume Node Filesystem When Volume Is Detaching and Attaching FOR ${i} IN RANGE ${LOOP_COUNT} And Detach volume 0 + And Wait for volume 0 detached And Attach volume 0 And Wait for volume 0 healthy And Check volume 0 data is intact diff --git a/e2e/tests/negative/stress_memory.robot b/e2e/tests/negative/stress_memory.robot index 91b9197777..6f3a5c6b90 100644 --- a/e2e/tests/negative/stress_memory.robot +++ b/e2e/tests/negative/stress_memory.robot @@ -43,6 +43,7 @@ Stress Volume Node Memory When Volume Is Detaching and Attaching When Stress memory of node with volume 0 And Detach volume 0 + And Wait for volume 0 detached And Attach volume 0 And Wait for volume 0 healthy Then Check volume 0 data is intact diff --git a/e2e/tests/regression/test_backing_image.robot b/e2e/tests/regression/test_backing_image.robot index b3ab113495..7eb1564a86 100644 --- a/e2e/tests/regression/test_backing_image.robot +++ b/e2e/tests/regression/test_backing_image.robot @@ -30,6 +30,7 @@ Test Backing Image Basic Operation And Verify clean up backing image bi from a disk will fail And Verify delete backing image bi will fail And Detach volume 0 + And Wait for volume 0 detached And Delete volume 0 And Clean up backing image bi from a disk And Delete backing image bi diff --git a/e2e/tests/regression/test_backup.robot b/e2e/tests/regression/test_backup.robot index 14cbc35cea..7a793d0131 100644 --- a/e2e/tests/regression/test_backup.robot +++ b/e2e/tests/regression/test_backup.robot @@ -102,6 +102,7 @@ Test Incremental Restore And Wait for volume 3 healthy And Check volume 3 data is backup 2 And Detach volume 3 + And Wait for volume 3 detached When Create persistentvolume for volume 3 And Create persistentvolumeclaim for volume 3 diff --git a/e2e/tests/regression/test_basic.robot b/e2e/tests/regression/test_basic.robot index 9274923182..b0791e30c9 100644 --- a/e2e/tests/regression/test_basic.robot +++ b/e2e/tests/regression/test_basic.robot @@ -53,6 +53,7 @@ Test Volume Basic Then Check volume 0 data is intact And Detach volume 0 + And Wait for volume 0 detached And Delete volume 0 Test Snapshot @@ -80,11 +81,13 @@ Test Snapshot And Check volume 0 data is data 2 When Detach volume 0 + And Wait for volume 0 detached And Attach volume 0 in maintenance mode And Wait for volume 0 healthy And Revert volume 0 to snapshot 1 And Detach volume 0 + And Wait for volume 0 detached And Attach volume 0 And Wait for volume 0 healthy Then Check volume 0 data is data 1 diff --git a/e2e/tests/regression/test_settings.robot b/e2e/tests/regression/test_settings.robot index 1bcffdc303..bb28480533 100644 --- a/e2e/tests/regression/test_settings.robot +++ b/e2e/tests/regression/test_settings.robot @@ -64,6 +64,7 @@ Test Setting Concurrent Rebuild Limit # Test the setting won't intervene normal attachment. Given Set setting concurrent-replica-rebuild-per-node-limit to 1 When Detach volume 1 + And Wait for volume 1 detached And Delete volume 0 replica on replica node And Wait until volume 0 replica rebuilding started on replica node And Attach volume 1 diff --git a/e2e/tests/regression/test_v2.robot b/e2e/tests/regression/test_v2.robot index 137d7eb7c3..fddba633a7 100644 --- a/e2e/tests/regression/test_v2.robot +++ b/e2e/tests/regression/test_v2.robot @@ -32,6 +32,7 @@ Test V2 Volume Basic And Write data to volume 0 Then Check volume 0 data is intact And Detach volume 0 + And Wait for volume 0 detached And Delete volume 0 Degraded Volume Replica Rebuilding diff --git a/e2e/tests/regression/test_volume.robot b/e2e/tests/regression/test_volume.robot index bca2ad2266..f6954594d9 100644 --- a/e2e/tests/regression/test_volume.robot +++ b/e2e/tests/regression/test_volume.robot @@ -9,6 +9,8 @@ Resource ../keywords/longhorn.resource Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/setting.resource Resource ../keywords/workload.resource +Resource ../keywords/node.resource +Resource ../keywords/volume.resource Test Setup Set test environment Test Teardown Cleanup test resources @@ -42,3 +44,19 @@ Test RWX volume data integrity after CSI plugin pod restart ... longhorn-csi-plugin Then Check deployment 0 data in file data.txt is intact + +Test detached volume should not reattach after node eviction + [Tags] volume node-eviction + [Documentation] Test detached volume should not reattach after node eviction. + ... + ... Issue: https://github.com/longhorn/longhorn/issues/9781 + + Given Create volume 0 with dataEngine=${DATA_ENGINE} + And Attach volume 0 + And Wait for volume 0 healthy + + When Detach volume 0 + And Set node 1 with allowScheduling=false evictionRequested=true + + Then Wait for volume 0 detached + And Assert volume 0 remains detached for at least 60 seconds diff --git a/e2e/utilities/longhorn-install.sh b/e2e/utilities/longhorn-install.sh old mode 100644 new mode 100755 index 9cd0d428c2..eafdd6beb0 --- a/e2e/utilities/longhorn-install.sh +++ b/e2e/utilities/longhorn-install.sh @@ -13,6 +13,8 @@ source ../pipelines/utilities/longhorn_manifest.sh # create and clean tmpdir TMPDIR="/tmp/longhorn" LONGHORN_NAMESPACE="longhorn-system" +LONGHORN_REPO_DIR="${TMPDIR}/longhorn" +LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"} mkdir -p ${TMPDIR} rm -rf "${TMPDIR}/" @@ -23,19 +25,48 @@ install_longhorn_by_chart(){ wait_longhorn_status_running } +install_longhorn_stable_by_chart(){ + git clone --single-branch \ + --branch "${LONGHORN_STABLE_VERSION}" \ + "${LONGHORN_REPO_URI}" \ + "${LONGHORN_REPO_DIR}" + helm upgrade --install longhorn "${LONGHORN_REPO_DIR}/chart/" --namespace "${LONGHORN_NAMESPACE}" + wait_longhorn_status_running +} + +install_longhorn_stable_by_manifest(){ + LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} + LONGHORN_STABLE_MANIFEST_URL="https://raw.githubusercontent.com/longhorn/longhorn/${LONGHORN_STABLE_VERSION}/deploy/longhorn.yaml" + kubectl apply -f "${LONGHORN_STABLE_MANIFEST_URL}" + wait_longhorn_status_running +} + install_longhorn(){ create_longhorn_namespace install_backupstores if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then - LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"} - LONGHORN_REPO_DIR="${TMPDIR}/longhorn" install_longhorn_by_chart elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then generate_longhorn_yaml_manifest "${TF_VAR_tf_workspace}" install_longhorn_by_manifest "${TF_VAR_tf_workspace}/longhorn.yaml" fi setup_longhorn_ui_nodeport +} +install_longhorn_stable_version(){ + create_longhorn_namespace + install_backupstores + if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then + install_longhorn_stable_by_chart + elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then + install_longhorn_stable_by_manifest + fi + setup_longhorn_ui_nodeport } -install_longhorn +IS_INSTALL_STABLE_VERSION="${IS_INSTALL_STABLE_VERSION:-false}" +if [[ "${IS_INSTALL_STABLE_VERSION}" == "true" ]]; then + install_longhorn_stable_version +else + install_longhorn +fi diff --git a/pipelines/utilities/run_longhorn_e2e_test.sh b/pipelines/utilities/run_longhorn_e2e_test.sh index e75b7c3732..8cd6606fe8 100755 --- a/pipelines/utilities/run_longhorn_e2e_test.sh +++ b/pipelines/utilities/run_longhorn_e2e_test.sh @@ -42,6 +42,7 @@ run_longhorn_e2e_test(){ yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_INSTALL_METHOD", "value": "'${LONGHORN_INSTALL_METHOD}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_STABLE_VERSION", "value": "'${LONGHORN_STABLE_VERSION}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}` @@ -106,6 +107,7 @@ run_longhorn_e2e_test_out_of_cluster(){ -e CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE="${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}"\ -e CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE="${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}"\ -e LONGHORN_INSTALL_METHOD="${LONGHORN_INSTALL_METHOD}"\ + -e LONGHORN_STABLE_VERSION="${LONGHORN_STABLE_VERSION}"\ --mount source="vol-${IMAGE_NAME}",target=/tmp \ "${LONGHORN_TESTS_CUSTOM_IMAGE}" "${ROBOT_COMMAND_ARGS[@]}" docker stop "${CONTAINER_NAME}"