From 4c7c87b8e47bb77c1aa1bdb1615fac64d50f5429 Mon Sep 17 00:00:00 2001 From: Freddie Mather Date: Fri, 6 Sep 2024 10:46:19 +0000 Subject: [PATCH 1/5] 87 - updated config to include job def type --- test_harness/config/config.py | 5 ++++- test_harness/config/default_test_config.yaml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test_harness/config/config.py b/test_harness/config/config.py index 19762b9e..e130e282 100644 --- a/test_harness/config/config.py +++ b/test_harness/config/config.py @@ -3,7 +3,7 @@ """ import os -from typing import Optional +from typing import Optional, Literal from configparser import ConfigParser from pathlib import Path @@ -220,6 +220,7 @@ def parse_from_dict(self, test_config: dict[str, str | dict]) -> None: "finish_interval": `int` => 0, defaults to 30; "timeout": `int` => 0, defaults to 120; }, + "jobdef_type": `Literal`[`"uml"`, `"json"`], defaults to `"uml"`; } :type test_config: `dict`[`str`, `str` | `dict`] """ @@ -255,6 +256,7 @@ def set_default_config(self) -> None: self.sample_rate = 0 self.low_memory = False self.test_finish = {} + self.jobdef_type: Literal["uml", "json"] = "uml" def config_to_dict(self) -> dict: """Provide config as a dictionary""" @@ -267,6 +269,7 @@ def config_to_dict(self) -> dict: "sample_rate": self.sample_rate, "low_memory": self.low_memory, "test_finish": self.test_finish, + "jobdef_type": self.jobdef_type, } if self.type != "Functional": config_dict_to_return["performance_options"] = ( diff --git a/test_harness/config/default_test_config.yaml b/test_harness/config/default_test_config.yaml index e7227420..5fb2caf9 100644 --- a/test_harness/config/default_test_config.yaml +++ b/test_harness/config/default_test_config.yaml @@ -33,3 +33,5 @@ test_finish: metric_get_interval: 5 finish_interval: 30 timeout: 120 + +jobdef_type: "uml" \ No newline at end of file From b74bc2860fbac9950af1960dea2dc9b3d1e86065 Mon Sep 17 00:00:00 2001 From: Freddie Mather Date: Fri, 6 Sep 2024 10:47:01 +0000 Subject: [PATCH 2/5] 87 - updated functionality to include ability to use job def jsons --- test_harness/protocol_verifier/__init__.py | 9 +- .../protocol_verifier/send_job_defs.py | 103 ++++++++++++++++-- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/test_harness/protocol_verifier/__init__.py b/test_harness/protocol_verifier/__init__.py index ed44923c..f34c6691 100644 --- a/test_harness/protocol_verifier/__init__.py +++ b/test_harness/protocol_verifier/__init__.py @@ -16,7 +16,7 @@ generate_test_events_from_puml_files, get_test_events_from_test_file_jsons, ) -from test_harness.protocol_verifier.send_job_defs import send_job_defs_from_uml +from test_harness.protocol_verifier.send_job_defs import handle_send_job_defs from test_harness.protocol_verifier.testing_suite.base_test_classes import ( FunctionalTest, PerformanceTest, @@ -126,11 +126,12 @@ def puml_files_test( ) # send job definitions to pv - send_job_defs_from_uml( - url=harness_config.pv_send_job_defs_url, - uml_file_paths=puml_file_paths, + handle_send_job_defs( + file_paths=puml_file_paths, harness_config=harness_config, + file_type=test_config.jobdef_type, ) + logging.getLogger().info( "Waiting %ds for job defs to load", harness_config.pv_config_update_time, diff --git a/test_harness/protocol_verifier/send_job_defs.py b/test_harness/protocol_verifier/send_job_defs.py index 965fa94a..03315536 100644 --- a/test_harness/protocol_verifier/send_job_defs.py +++ b/test_harness/protocol_verifier/send_job_defs.py @@ -1,6 +1,7 @@ """Methods to send job defs to PV using uml file paths""" from io import BytesIO +from typing import Literal from test_harness.protocol_verifier.config.config import ProtocolVerifierConfig from test_harness.utils import create_file_io_file_name_tuple_with_file_path @@ -10,25 +11,58 @@ from test_harness.requests_th.send_config import post_config_form_upload -def send_job_defs_from_uml( - url: str, uml_file_paths: list[str], harness_config: ProtocolVerifierConfig +def handle_send_job_defs( + file_paths: list[str], + harness_config: ProtocolVerifierConfig, + file_type: Literal["json", "uml"], ) -> None: - """Method to send job defs from a list of uml file paths to an url with + """Method to handle sending job defs to an url with :class:`ProtocolVerifierConfig` :param url: The url to send the request for uploading job definitions :type url: `str` - :param uml_file_paths: A list of filepaths to uml file job definitions - :type uml_file_paths: `list`[`str`] + :param file_paths: A list of file paths to job definitions + :type file_paths: `list`[`str`] + :param harness_config: Config for the test harness + :type harness_config: :class:`ProtocolVerifierConfig` + :param file_type: The type of file to send job defs from + :type file_type: `Literal`["json", "uml"] + """ + match file_type: + case "json": + send_job_defs_from_json( + harness_config.pv_send_job_defs_url, file_paths, harness_config + ) + case "uml": + send_job_defs_from_uml( + harness_config.pv_send_job_defs_url, file_paths, harness_config + ) + case _: + raise ValueError(f"Invalid file type: {file_type}") + + +def send_job_defs( + url: str, job_defs: list[str], + file_paths: list[str], + harness_config: ProtocolVerifierConfig +) -> None: + """Method to send job defs from a list of job defs and file paths to an url + with :class:`ProtocolVerifierConfig` + + :param url: The url to send the request for uploading job definitions + :type url: `str` + :param job_defs: A list of job definitions as strings + :type job_defs: `list`[`str`] + :param file_paths: A list of file paths to job definitions + :type file_paths: `list`[`str`] :param harness_config: Config for the test harness :type harness_config: :class:`ProtocolVerifierConfig` """ - job_defs = get_job_defs_from_uml_files(uml_file_paths) file_io_file_name_tuples = [ create_file_io_file_name_tuple_with_file_path( - file_path.replace(".puml", ".json"), file_string + file_path, job_def ) - for file_path, file_string in zip(uml_file_paths, job_defs) + for job_def, file_path in zip(job_defs, file_paths) ] send_job_defs_from_file_io_file_name_tuples( file_io_file_name_tuples=file_io_file_name_tuples, @@ -38,6 +72,59 @@ def send_job_defs_from_uml( ) +def get_job_defs_from_jsons(json_file_paths: list[str]) -> list[str]: + """Method to get job defs from a list of json file paths + + :param json_file_paths: A list of file paths to json file job definitions + :type json_file_paths: `list`[`str`] + :return: A list of job definitions as strings + :rtype: `list`[`str`] + """ + job_defs: list[str] = [] + for json_file_path in json_file_paths: + with open(json_file_path, "r") as json_file: + job_defs.append(json_file.read()) + return job_defs + + +def send_job_defs_from_json( + url: str, json_file_paths: list[str], + harness_config: ProtocolVerifierConfig +) -> None: + """Method to send job defs from a list of json file paths to an url with + :class:`ProtocolVerifierConfig` + + :param url: The url to send the request for uploading job definitions + :type url: `str` + :param json_file_paths: A list of file paths to json file job definitions + :type json_file_paths: `list`[`str`] + :param harness_config: Config for the test harness + :type harness_config: :class:`ProtocolVerifierConfig` + """ + job_defs = get_job_defs_from_jsons(json_file_paths) + send_job_defs(url, job_defs, json_file_paths, harness_config) + + +def send_job_defs_from_uml( + url: str, uml_file_paths: list[str], harness_config: ProtocolVerifierConfig +) -> None: + """Method to send job defs from a list of uml file paths to an url with + :class:`ProtocolVerifierConfig` + + :param url: The url to send the request for uploading job definitions + :type url: `str` + :param uml_file_paths: A list of filepaths to uml file job definitions + :type uml_file_paths: `list`[`str`] + :param harness_config: Config for the test harness + :type harness_config: :class:`ProtocolVerifierConfig` + """ + job_defs = get_job_defs_from_uml_files(uml_file_paths) + converted_file_paths = [ + file_path.replace(".puml", ".json") for file_path in uml_file_paths + ] + send_job_defs(url, job_defs, converted_file_paths, harness_config) + + def send_job_defs_from_file_io_file_name_tuples( file_io_file_name_tuples: list[tuple[BytesIO, str]], url: str, From 50b72e2960bcdfc6e61644402cd5f7b545c9eb4e Mon Sep 17 00:00:00 2001 From: Freddie Mather Date: Fri, 6 Sep 2024 10:47:22 +0000 Subject: [PATCH 3/5] 87 - test updated functionality for job def jsons --- .../tests/test_files/test_uml_1_jobdef.json | 25 +++++ .../tests/test_send_job_defs.py | 99 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 test_harness/protocol_verifier/tests/test_files/test_uml_1_jobdef.json diff --git a/test_harness/protocol_verifier/tests/test_files/test_uml_1_jobdef.json b/test_harness/protocol_verifier/tests/test_files/test_uml_1_jobdef.json new file mode 100644 index 00000000..bf4a1c08 --- /dev/null +++ b/test_harness/protocol_verifier/tests/test_files/test_uml_1_jobdef.json @@ -0,0 +1,25 @@ +{ + "JobDefinitionName": "test_uml_1", + "Events": [ + { + "EventName": "A", + "OccurrenceId": 0, + "SequenceName": "test_uml_1", + "Application": "default_application_name", + "SequenceStart": true + }, + { + "EventName": "B", + "OccurrenceId": 0, + "SequenceName": "test_uml_1", + "Application": "default_application_name", + "SequenceEnd": true, + "PreviousEvents": [ + { + "PreviousEventName": "A", + "PreviousOccurrenceId": 0 + } + ] + } + ] +} diff --git a/test_harness/protocol_verifier/tests/test_send_job_defs.py b/test_harness/protocol_verifier/tests/test_send_job_defs.py index 873c1eeb..79bfe182 100644 --- a/test_harness/protocol_verifier/tests/test_send_job_defs.py +++ b/test_harness/protocol_verifier/tests/test_send_job_defs.py @@ -4,15 +4,21 @@ from pathlib import Path import os from configparser import ConfigParser +from unittest.mock import patch import responses +from responses.matchers import multipart_matcher import pytest from test_harness.config.config import HarnessConfig +from test_harness.protocol_verifier.config.config import ProtocolVerifierConfig from test_harness.utils import create_file_io_file_name_tuple from test_harness.protocol_verifier.send_job_defs import ( send_job_defs_from_file_io_file_name_tuples, send_job_defs_from_uml, + get_job_defs_from_jsons, + send_job_defs_from_json, + handle_send_job_defs, ) # test file resources folder @@ -96,3 +102,96 @@ def test_send_job_defs_from_uml() -> None: uml_file_paths=[test_uml_file_path_1, test_uml_file_path_2], harness_config=harness_config, ) + +class TestSendJobDefsFromJson: + @staticmethod + def json_string() -> str: + return ( + '{\n' + ' "JobDefinitionName": "test_uml_1",\n' + ' "Events": [\n' + ' {\n' + ' "EventName": "A",\n' + ' "OccurrenceId": 0,\n' + ' "SequenceName": "test_uml_1",\n' + ' "Application": "default_application_name",\n' + ' "SequenceStart": true\n' + ' },\n' + ' {\n' + ' "EventName": "B",\n' + ' "OccurrenceId": 0,\n' + ' "SequenceName": "test_uml_1",\n' + ' "Application": "default_application_name",\n' + ' "SequenceEnd": true,\n' + ' "PreviousEvents": [\n' + ' {\n' + ' "PreviousEventName": "A",\n' + ' "PreviousOccurrenceId": 0\n' + ' }\n' + ' ]\n' + ' }\n' + ' ]\n' + '}\n' + ) + + + def test_get_job_defs_from_jsons(self) -> None: + """Tests get_job_defs_from_jsons""" + test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + test_json_file_path_2 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + job_defs = get_job_defs_from_jsons( + json_file_paths=[test_json_file_path_1, test_json_file_path_2] + ) + json_string = self.json_string + assert all(json_string == job_def for job_def in job_defs) + + @responses.activate + def test_send_job_defs_from_jsons(self) -> None: + url = "http://mockserver.com/job-definitions" + responses.post( + url, status=200, + match=[multipart_matcher( + files={ + "upload": ( + "test_uml_1_jobdef.json", + self.json_string().encode("utf-8"), + "application/octet-stream" + ) + } + )] + ) + harness_config = HarnessConfig(config_parser) + test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + send_job_defs_from_json( + url=url, + json_file_paths=[test_json_file_path_1], + harness_config=harness_config + ) + + +def test_handle_send_job_defs() -> None: + """Tests handle_send_job_defs""" + harness_config = ProtocolVerifierConfig(config_parser) + test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + test_uml_file_path_1 = os.path.join(test_file_resources, "test_uml_1.puml") + with patch("test_harness.protocol_verifier.send_job_defs.send_job_defs_from_json") as mock: + handle_send_job_defs( + file_paths=[test_json_file_path_1, test_uml_file_path_1], + harness_config=harness_config, + file_type="json" + ) + mock.assert_called_once() + with patch("test_harness.protocol_verifier.send_job_defs.send_job_defs_from_uml") as mock: + handle_send_job_defs( + file_paths=[test_json_file_path_1, test_uml_file_path_1], + harness_config=harness_config, + file_type="uml" + ) + mock.assert_called_once() + with pytest.raises(ValueError) as e_info: + handle_send_job_defs( + file_paths=[test_json_file_path_1, test_uml_file_path_1], + harness_config=harness_config, + file_type="invalid" + ) + assert e_info.value.args[0] == "Invalid file type: invalid" \ No newline at end of file From a36fd05f3059df63e252f8327d11a5c8c39ba523 Mon Sep 17 00:00:00 2001 From: Freddie Mather Date: Fri, 6 Sep 2024 10:47:48 +0000 Subject: [PATCH 4/5] 87 - Update end to end tests to test using jobdef json --- .github/workflows/end-to-end-test.yml | 47 ++++++++++++++++++ ...url-commands-functional-test-jobdefjson.sh | 16 ++++++ .../test_files/test_jobdefjson.zip | Bin 0 -> 1412 bytes 3 files changed, 63 insertions(+) create mode 100755 scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh create mode 100644 tests/test_harness/test_files/test_jobdefjson.zip diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml index 80402048..57959d6c 100644 --- a/.github/workflows/end-to-end-test.yml +++ b/.github/workflows/end-to-end-test.yml @@ -77,6 +77,8 @@ jobs: docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully" + rm ./munin/deploy/config/job_definitions/*.json + - name: Check for performance test failures run: | # Inspect report output for failures of performance test @@ -115,6 +117,8 @@ jobs: curl 127.0.0.1:8800/isTestRunning | grep 'false' docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully" + + rm ./munin/deploy/config/job_definitions/*.json - name: Check for functional test failures run: | @@ -139,6 +143,49 @@ jobs: echo "No test failures." exit 0 fi + + - name: Functional test with jobdef json + run: | + echo "Running a functional test" + timeout 1m ./scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh + + # this should return true as tests are running + echo "Testing that tests are running" + curl 127.0.0.1:8800/isTestRunning | grep 'true' + + # It takes this long to get any meaningful output from the test harness + # It's 60 seconds for the test harness to start the job ... + # ... and 10 seconds to run the job ... + # ... and 60 seconds to ensure the test is finished ... + # ... and another 50 seconds for grace time in starting up/waiting for logs/waiting for calculations + echo "Sleeping to give the test harness time to work" + date + sleep 120 + date + curl 127.0.0.1:8800/isTestRunning | grep 'false' + + docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully" + rm ./munin/deploy/config/job_definitions/*.json + + - name: Check for functional test failures job def json + run: | + # Inspect report output for failures of functional test + failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/functional_test_jobdef/Results.xml) + + + + # Print the result + echo "failures: $failures" + + # Check if failures is greater than 0 + if [ "$failures" -ne 0 ]; then + echo "There are functional test failures!" + exit 1 + else + echo "No test failures." + exit 0 + fi + - name: Run tear down procedure run: | diff --git a/scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh b/scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh new file mode 100755 index 00000000..b4265036 --- /dev/null +++ b/scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# We do a while loop here to keep requesting the named-zip-files endpoint until the test harness is up and running +echo "Checking if test harness is up and running by uploading test files" +while ! curl --location --request POST 'http://127.0.0.1:8800/upload/named-zip-files' --form 'functional_test_jobdef=@"tests/test_harness/test_files/test_jobdefjson.zip"' -s -o /dev/null -w "%{http_code}" | grep -q 200; do + echo tried to do a curl request to named-zip-files endpoint but it failed, trying again in 1 second + echo printing test harness logs to see what is going on + docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | tail + sleep 1 +done +echo "Uploading test config" +curl -X POST -d '{"TestName": "functional_test_jobdef", "TestConfig":{"type":"Functional"}}' -H 'Content-Type: application/json' 'http://127.0.0.1:8800/startTest' +echo "Waiting for functional test to start" +sleep 1 \ No newline at end of file diff --git a/tests/test_harness/test_files/test_jobdefjson.zip b/tests/test_harness/test_files/test_jobdefjson.zip new file mode 100644 index 0000000000000000000000000000000000000000..f129819687726a5f2b288164eb2cbe47d445c944 GIT binary patch literal 1412 zcmWIWW@h1H0D)D(YLQ?Dln`K$VJJy0E{V^|PfAHm%PP*#(+>^dWMFQ%c`L0Ch)XND z85mh!Ff%ZKi2$HUB0#+y3^PO3BJaJuzqbg;n+n7VSWE(&oSdJRmYJ?snV6dcHd*oR ztuzouGugM#k+0c+hvj``>s0Xz(NEdGu&5muKFG8?CDKZZn0}DSMw4^9T6_5-JW1PY-k_(BO} z^6uZ=^Nko87*c?NXof44kW2=LT4`=hykUH5S!!NMu^uQ|z+nfn4@<0^4fgeCG8EYR zS^LDil`ps5%{}ca%pbSM!zSazGWp9(()Oq9R%!Wg|7}x4gVbX2=O#11-z)iL72AKz zaME-)@pIEB_#8if`;@?I=WWmL-1P96aeBw$DM~($O4GQTO|J6HoMQO2WsyUH(N2N; ztFvCrXL;dtF+6qRyvYfir8Cyv(tZ0YxWv5iL}x+R^A64j9gUYfq%tR-*>dqf%d>Au zO%GOAY*=+`A>%=nK7B#Ybj{h9a=M<1sy8KnD_Iwu8(p=yDgV`9ULA|8(-;0=5dGR9 zJ7xAs`@)aUq{XME?hKxwsAk!x_bEeR8%v|w6|PrBT+hPwxPGaNpI`EpY1+P*yKmop za`$APT!ej8r|```Kca5u*jfH-{iQv9GGAQv3SN5GXQ`A_yE6Nx-N8VYFl%;LXS+$BZkpNx-w+TSpKRGwZQJvL04b zab-M+slXJ*u%yuoi>a8|5NHdQEQsuO%q)TIb}^u3=xzt|Q9=o53zozRu?6B6% Date: Fri, 6 Sep 2024 15:27:05 +0000 Subject: [PATCH 5/5] 87 - add test for run app and end to end test a performance test with correct data --- .github/workflows/end-to-end-test.yml | 56 ++++++------ ...l-commands-performance-test-jobdefjson.sh} | 6 +- .../tests/test_protocol_verifier__init__.py | 8 +- .../tests/test_send_job_defs.py | 84 +++++++++++------- .../test_files/test_jobdefjson.zip | Bin 1412 -> 2261 bytes tests/test_harness/test_run_app.py | 25 +++++- 6 files changed, 108 insertions(+), 71 deletions(-) rename scripts/{end-to-end-curl-commands-functional-test-jobdefjson.sh => end-to-end-curl-commands-performance-test-jobdefjson.sh} (57%) diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml index 57959d6c..28bb0189 100644 --- a/.github/workflows/end-to-end-test.yml +++ b/.github/workflows/end-to-end-test.yml @@ -95,11 +95,11 @@ jobs: echo "No test failures." exit 0 fi - - - name: Run functional test + + - name: Run performance test job def json run: | - echo "Running a functional test" - timeout 1m ./scripts/end-to-end-curl-commands-functional-test.sh + echo "Running a performance test" + timeout 1m ./scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh # this should return true as tests are running echo "Testing that tests are running" @@ -117,37 +117,30 @@ jobs: curl 127.0.0.1:8800/isTestRunning | grep 'false' docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully" - + rm ./munin/deploy/config/job_definitions/*.json - - name: Check for functional test failures + - name: Check for performance test failures job def json run: | - # Inspect report output for failures of functional test - failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/functional_test/Results.xml) - - # These are known failures, so we will discount them - stacked_solution_failures=$(xmllint --xpath 'string(//testsuite[@name="simple_XOR_job.False.StackedSolutions"]/@failures)' ./report_output/functional_test/Results.xml) - - # Calculate non stacked solution failures - non_stacked_solution_failures=$(($failures - $stacked_solution_failures)) + # Inspect report output for failures of performance test + failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/performance_test_jobdef/Report.xml) # Print the result - echo "Known stacked solution failures: $stacked_solution_failures" - echo "Non stacked solution failures: $non_stacked_solution_failures" + echo "Performance test failures: $failures" - # Check if non stacked solution failures is greater than 0 - if [ "$non_stacked_solution_failures" -ne 0 ]; then - echo "There are functional test failures!" + # Check if failures is not zero + if [ "$failures" -ne 0 ]; then + echo "There are performance test failures!" exit 1 else echo "No test failures." exit 0 fi - - - name: Functional test with jobdef json + + - name: Run functional test run: | echo "Running a functional test" - timeout 1m ./scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh + timeout 1m ./scripts/end-to-end-curl-commands-functional-test.sh # this should return true as tests are running echo "Testing that tests are running" @@ -165,28 +158,33 @@ jobs: curl 127.0.0.1:8800/isTestRunning | grep 'false' docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully" + rm ./munin/deploy/config/job_definitions/*.json - - name: Check for functional test failures job def json + - name: Check for functional test failures run: | # Inspect report output for failures of functional test - failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/functional_test_jobdef/Results.xml) + failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/functional_test/Results.xml) + # These are known failures, so we will discount them + stacked_solution_failures=$(xmllint --xpath 'string(//testsuite[@name="simple_XOR_job.False.StackedSolutions"]/@failures)' ./report_output/functional_test/Results.xml) + # Calculate non stacked solution failures + non_stacked_solution_failures=$(($failures - $stacked_solution_failures)) # Print the result - echo "failures: $failures" + echo "Known stacked solution failures: $stacked_solution_failures" + echo "Non stacked solution failures: $non_stacked_solution_failures" - # Check if failures is greater than 0 - if [ "$failures" -ne 0 ]; then + # Check if non stacked solution failures is greater than 0 + if [ "$non_stacked_solution_failures" -ne 0 ]; then echo "There are functional test failures!" exit 1 else echo "No test failures." exit 0 fi - - + - name: Run tear down procedure run: | docker compose -f ./munin/deploy/docker-compose.prod.yml down diff --git a/scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh b/scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh similarity index 57% rename from scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh rename to scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh index b4265036..edc5ffb7 100755 --- a/scripts/end-to-end-curl-commands-functional-test-jobdefjson.sh +++ b/scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh @@ -4,13 +4,13 @@ set -e # We do a while loop here to keep requesting the named-zip-files endpoint until the test harness is up and running echo "Checking if test harness is up and running by uploading test files" -while ! curl --location --request POST 'http://127.0.0.1:8800/upload/named-zip-files' --form 'functional_test_jobdef=@"tests/test_harness/test_files/test_jobdefjson.zip"' -s -o /dev/null -w "%{http_code}" | grep -q 200; do +while ! curl --location --request POST 'http://127.0.0.1:8800/upload/named-zip-files' --form 'performance_test_jobdef=@"tests/test_harness/test_files/test_jobdefjson.zip"' -s -o /dev/null -w "%{http_code}" | grep -q 200; do echo tried to do a curl request to named-zip-files endpoint but it failed, trying again in 1 second echo printing test harness logs to see what is going on docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | tail sleep 1 done echo "Uploading test config" -curl -X POST -d '{"TestName": "functional_test_jobdef", "TestConfig":{"type":"Functional"}}' -H 'Content-Type: application/json' 'http://127.0.0.1:8800/startTest' -echo "Waiting for functional test to start" +curl -X POST -d '{"TestName": "performance_test_jobdef", "TestConfig":{"type":"Performance", "performance_options": {"num_files_per_sec":10}}}' -H 'Content-Type: application/json' 'http://127.0.0.1:8800/startTest' +echo "Waiting for performance test to start" sleep 1 \ No newline at end of file diff --git a/test_harness/protocol_verifier/tests/test_protocol_verifier__init__.py b/test_harness/protocol_verifier/tests/test_protocol_verifier__init__.py index 012e32e1..80014675 100644 --- a/test_harness/protocol_verifier/tests/test_protocol_verifier__init__.py +++ b/test_harness/protocol_verifier/tests/test_protocol_verifier__init__.py @@ -108,12 +108,16 @@ uuid4hex = re.compile("[0-9a-f]{12}4[0-9a-f]{3}[89ab][0-9a-f]{15}\\Z", re.I) +@pytest.mark.parametrize("jobdef_type", ["json", "uml"]) @responses.activate -def test_puml_files_test() -> None: +def test_puml_files_test(jobdef_type: Literal["json", "uml"]) -> None: """Tests method `puml_test_files`""" harness_config = ProtocolVerifierConfig(config_parser) test_config = TestConfig() - test_config.parse_from_dict({"event_gen_options": {"invalid": False}}) + test_config.parse_from_dict({ + "event_gen_options": {"invalid": False}, + "jobdef_type": jobdef_type, + }) with mock_pv_http_interface(harness_config): puml_files_test( puml_file_paths=[test_file_path], diff --git a/test_harness/protocol_verifier/tests/test_send_job_defs.py b/test_harness/protocol_verifier/tests/test_send_job_defs.py index 79bfe182..0d204095 100644 --- a/test_harness/protocol_verifier/tests/test_send_job_defs.py +++ b/test_harness/protocol_verifier/tests/test_send_job_defs.py @@ -103,95 +103,113 @@ def test_send_job_defs_from_uml() -> None: harness_config=harness_config, ) + class TestSendJobDefsFromJson: + """Tests for send_job_defs_from_json""" @staticmethod def json_string() -> str: + """Returns a json string""" return ( - '{\n' + "{\n" ' "JobDefinitionName": "test_uml_1",\n' ' "Events": [\n' - ' {\n' + " {\n" ' "EventName": "A",\n' ' "OccurrenceId": 0,\n' ' "SequenceName": "test_uml_1",\n' ' "Application": "default_application_name",\n' ' "SequenceStart": true\n' - ' },\n' - ' {\n' + " },\n" + " {\n" ' "EventName": "B",\n' ' "OccurrenceId": 0,\n' ' "SequenceName": "test_uml_1",\n' ' "Application": "default_application_name",\n' ' "SequenceEnd": true,\n' ' "PreviousEvents": [\n' - ' {\n' + " {\n" ' "PreviousEventName": "A",\n' ' "PreviousOccurrenceId": 0\n' - ' }\n' - ' ]\n' - ' }\n' - ' ]\n' - '}\n' + " }\n" + " ]\n" + " }\n" + " ]\n" + "}\n" ) - def test_get_job_defs_from_jsons(self) -> None: """Tests get_job_defs_from_jsons""" - test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") - test_json_file_path_2 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + test_json_file_path_1 = os.path.join( + test_file_resources, "test_uml_1_jobdef.json" + ) + test_json_file_path_2 = os.path.join( + test_file_resources, "test_uml_1_jobdef.json" + ) job_defs = get_job_defs_from_jsons( json_file_paths=[test_json_file_path_1, test_json_file_path_2] ) - json_string = self.json_string + json_string = self.json_string() assert all(json_string == job_def for job_def in job_defs) @responses.activate def test_send_job_defs_from_jsons(self) -> None: + """Tests send_job_defs_from_jsons""" url = "http://mockserver.com/job-definitions" responses.post( - url, status=200, - match=[multipart_matcher( - files={ - "upload": ( - "test_uml_1_jobdef.json", - self.json_string().encode("utf-8"), - "application/octet-stream" - ) - } - )] + url, + status=200, + match=[ + multipart_matcher( + files={ + "upload": ( + "test_uml_1_jobdef.json", + self.json_string().encode("utf-8"), + "application/octet-stream", + ) + } + ) + ], ) harness_config = HarnessConfig(config_parser) - test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + test_json_file_path_1 = os.path.join( + test_file_resources, "test_uml_1_jobdef.json" + ) send_job_defs_from_json( url=url, json_file_paths=[test_json_file_path_1], - harness_config=harness_config + harness_config=harness_config, ) def test_handle_send_job_defs() -> None: """Tests handle_send_job_defs""" harness_config = ProtocolVerifierConfig(config_parser) - test_json_file_path_1 = os.path.join(test_file_resources, "test_uml_1_jobdef.json") + test_json_file_path_1 = os.path.join( + test_file_resources, "test_uml_1_jobdef.json" + ) test_uml_file_path_1 = os.path.join(test_file_resources, "test_uml_1.puml") - with patch("test_harness.protocol_verifier.send_job_defs.send_job_defs_from_json") as mock: + with patch( + "test_harness.protocol_verifier.send_job_defs.send_job_defs_from_json" + ) as mock: handle_send_job_defs( file_paths=[test_json_file_path_1, test_uml_file_path_1], harness_config=harness_config, - file_type="json" + file_type="json", ) mock.assert_called_once() - with patch("test_harness.protocol_verifier.send_job_defs.send_job_defs_from_uml") as mock: + with patch( + "test_harness.protocol_verifier.send_job_defs.send_job_defs_from_uml" + ) as mock: handle_send_job_defs( file_paths=[test_json_file_path_1, test_uml_file_path_1], harness_config=harness_config, - file_type="uml" + file_type="uml", ) mock.assert_called_once() with pytest.raises(ValueError) as e_info: handle_send_job_defs( file_paths=[test_json_file_path_1, test_uml_file_path_1], harness_config=harness_config, - file_type="invalid" + file_type="invalid", ) - assert e_info.value.args[0] == "Invalid file type: invalid" \ No newline at end of file + assert e_info.value.args[0] == "Invalid file type: invalid" diff --git a/tests/test_harness/test_files/test_jobdefjson.zip b/tests/test_harness/test_files/test_jobdefjson.zip index f129819687726a5f2b288164eb2cbe47d445c944..d6302a4664173686be1fc02a6d3b7f86786a56c3 100644 GIT binary patch delta 1092 zcmZqSzADHQ;LXg!#Q*|a1rvGH*}rhzPUGReJvo6wZnQlJ(IRsbdBCL1yeOBEF5 zr)B1(#uu057p3ZlhHx@4t8m{=GY4yx5(H^tdp7O10#K785bJ?7)uU=ENi8mkhYIN> z7ni{e0U1KI`kHdx`Vp8Q-Ty7Yt1v7+nf9FnVBgF_2-?*@hej zkT*nt_HZyPD^`olH1@Gc2lCp0Sb;%?0c=xN{=|H_`t5?Z(?A%_eAlyvd`yNsEC=k? zZ}`+>(=e;)p3$9Wddm~s3pUxRi5eK4Jgfx#3M zev=)T`6u6I;jYihPfAHm)5|K(&jSYeX>Q3~$bDKG(=dZYGzK*xj#Yw>1DMCcfY~yW6=W*_0j6IT delta 348 zcmcaA*uu>d;LXg!#Q*}Uf+zB*Gd4_~&m=b4frX2AW~f@^y|?%G76Ij^PIhF}saL#v zD-DE8E4UdLSza(RFn|f)K1aT010I(5m9107FGN3O|H8W0UBt(biCtBC$v?U97>jpr z)@=Fl_K-;YS+ None: """Function to run performance test using requests for uploaded zip file functionality @@ -215,7 +223,7 @@ def run_performance_test_requests_zip_file_upload( # this will post the file under the name "upload" response = post_config_form_upload( file_bytes_file_names=[ - (open(test_file_zip_path, "rb"), "test_zip_file.zip") + (open(zip_file_path, "rb"), zip_file_name) ], url="http://localhost:8800/upload/named-zip-files", )[2] @@ -236,8 +244,15 @@ def run_performance_test_requests_zip_file_upload( time.sleep(1) +@pytest.mark.parametrize("zip_file_path, zip_file_name, jobdef_file_name", [ + (test_file_zip_path, "test_zip_file.zip", "test_uml_1.puml"), + (test_file_zip_path_2, "test_jobdefjson.zip", "test_uml_1_jobdef.json"), + +]) @responses.activate -def test_run_harness_app_uploaded_zip_file() -> None: +def test_run_harness_app_uploaded_zip_file( + zip_file_path: str, zip_file_name: str, jobdef_file_name: str +) -> None: """Test the `run_harness_app` function. This function sets up a test environment for the `run_harness_app` @@ -299,7 +314,9 @@ def reception_log_call_back( ) thread_2 = Thread( target=run_performance_test_requests_zip_file_upload, - args=(response_results,), + args=( + response_results, zip_file_path, zip_file_name + ), ) thread_1.start() time.sleep(5) @@ -316,7 +333,7 @@ def reception_log_call_back( ) for folder, file in zip( ["uml_file_store", "test_file_store", "profile_store"], - ["test_uml_1.puml", "test_uml_1_events.json", "test_profile.csv"], + [jobdef_file_name, "test_uml_1_events.json", "test_profile.csv"], ): path = os.path.join(test_output_path, folder, file) assert os.path.exists(path)