87 update test harness to use job defs rather than uml

.github/workflows/end-to-end-test.yml

@@ -77,6 +77,8 @@ jobs:
 
           docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
 
+          rm ./munin/deploy/config/job_definitions/*.json
+
       - name: Check for performance test failures
         run: |
           # Inspect report output for failures of performance test
@@ -93,6 +95,47 @@ jobs:
             echo "No test failures."
             exit 0
           fi
+
+      - name: Run performance test job def json
+        run: |
+          echo "Running a performance test"
+          timeout 1m ./scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh
+
+          # this should return true as tests are running
+          echo "Testing that tests are running"
+          curl 127.0.0.1:8800/isTestRunning | grep 'true'
+
+          # It takes this long to get any meaningful output from the test harness
+          # It's 60 seconds for the test harness to start the job ...
+          # ... and 10 seconds to run the job ...
+          # ... and 60 seconds to ensure the test is finished ...
+          # ... and another 50 seconds for grace time in starting up/waiting for logs/waiting for calculations
+          echo "Sleeping to give the test harness time to work"
+          date
+          sleep 120
+          date
+          curl 127.0.0.1:8800/isTestRunning | grep 'false'
+
+          docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
+
+          rm ./munin/deploy/config/job_definitions/*.json
+
+      - name: Check for performance test failures job def json
+        run: |
+          # Inspect report output for failures of performance test
+          failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/performance_test_jobdef/Report.xml)
+
+          # Print the result
+          echo "Performance test failures: $failures"
+
+          # Check if failures is not zero
+          if [ "$failures" -ne 0 ]; then
+            echo "There are performance test failures!"
+            exit 1
+          else
+            echo "No test failures."
+            exit 0
+          fi
 
       - name: Run functional test
         run: |
@@ -115,6 +158,8 @@ jobs:
           curl 127.0.0.1:8800/isTestRunning | grep 'false'
 
           docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
+
+          rm ./munin/deploy/config/job_definitions/*.json
 
       - name: Check for functional test failures
         run: |
@@ -139,7 +184,7 @@ jobs:
             echo "No test failures."
             exit 0
           fi
-
+        
       - name: Run tear down procedure
         run: |
           docker compose -f ./munin/deploy/docker-compose.prod.yml down

scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+# We do a while loop here to keep requesting the named-zip-files endpoint until the test harness is up and running
+echo "Checking if test harness is up and running by uploading test files"
+while ! curl --location --request POST 'http://127.0.0.1:8800/upload/named-zip-files' --form 'performance_test_jobdef=@"tests/test_harness/test_files/test_jobdefjson.zip"' -s -o /dev/null -w "%{http_code}" | grep -q 200; do
+    echo tried to do a curl request to named-zip-files endpoint but it failed, trying again in 1 second
+    echo printing test harness logs to see what is going on
+    docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | tail
+    sleep 1
+done
+echo "Uploading test config"
+curl -X POST -d '{"TestName": "performance_test_jobdef", "TestConfig":{"type":"Performance", "performance_options": {"num_files_per_sec":10}}}' -H 'Content-Type: application/json' 'http://127.0.0.1:8800/startTest'
+echo "Waiting for performance test to start"
+sleep 1

test_harness/config/config.py

@@ -3,7 +3,7 @@
 """
 
 import os
-from typing import Optional
+from typing import Optional, Literal
 from configparser import ConfigParser
 from pathlib import Path
 
@@ -220,6 +220,7 @@ def parse_from_dict(self, test_config: dict[str, str | dict]) -> None:
                 "finish_interval": `int` => 0, defaults to 30;
                 "timeout": `int` => 0, defaults to 120;
             },
+            "jobdef_type": `Literal`[`"uml"`, `"json"`], defaults to `"uml"`;
         }
         :type test_config: `dict`[`str`, `str` | `dict`]
         """
@@ -255,6 +256,7 @@ def set_default_config(self) -> None:
         self.sample_rate = 0
         self.low_memory = False
         self.test_finish = {}
+        self.jobdef_type: Literal["uml", "json"] = "uml"
 
     def config_to_dict(self) -> dict:
         """Provide config as a dictionary"""
@@ -267,6 +269,7 @@ def config_to_dict(self) -> dict:
             "sample_rate": self.sample_rate,
             "low_memory": self.low_memory,
             "test_finish": self.test_finish,
+            "jobdef_type": self.jobdef_type,
         }
         if self.type != "Functional":
             config_dict_to_return["performance_options"] = (

test_harness/config/default_test_config.yaml

@@ -33,3 +33,5 @@ test_finish:
   metric_get_interval: 5
   finish_interval: 30
   timeout: 120
+
+jobdef_type: "uml"

test_harness/protocol_verifier/__init__.py

@@ -16,7 +16,7 @@
     generate_test_events_from_puml_files,
     get_test_events_from_test_file_jsons,
 )
-from test_harness.protocol_verifier.send_job_defs import send_job_defs_from_uml
+from test_harness.protocol_verifier.send_job_defs import handle_send_job_defs
 from test_harness.protocol_verifier.testing_suite.base_test_classes import (
     FunctionalTest,
     PerformanceTest,
@@ -126,11 +126,12 @@ def puml_files_test(
             )
 
     # send job definitions to pv
-    send_job_defs_from_uml(
-        url=harness_config.pv_send_job_defs_url,
-        uml_file_paths=puml_file_paths,
+    handle_send_job_defs(
+        file_paths=puml_file_paths,
         harness_config=harness_config,
+        file_type=test_config.jobdef_type,
     )
+
     logging.getLogger().info(
         "Waiting %ds for job defs to load",
         harness_config.pv_config_update_time,

test_harness/protocol_verifier/send_job_defs.py

@@ -1,6 +1,7 @@
 """Methods to send job defs to PV using uml file paths"""
 
 from io import BytesIO
+from typing import Literal
 
 from test_harness.protocol_verifier.config.config import ProtocolVerifierConfig
 from test_harness.utils import create_file_io_file_name_tuple_with_file_path
@@ -10,25 +11,58 @@
 from test_harness.requests_th.send_config import post_config_form_upload
 
 
-def send_job_defs_from_uml(
-    url: str, uml_file_paths: list[str], harness_config: ProtocolVerifierConfig
+def handle_send_job_defs(
+    file_paths: list[str],
+    harness_config: ProtocolVerifierConfig,
+    file_type: Literal["json", "uml"],
 ) -> None:
-    """Method to send job defs from a list of uml file paths to an url with
+    """Method to handle sending job defs to an url with
     :class:`ProtocolVerifierConfig`
 
     :param url: The url to send the request for uploading job definitions
     :type url: `str`
-    :param uml_file_paths: A list of filepaths to uml file job definitions
-    :type uml_file_paths: `list`[`str`]
+    :param file_paths: A list of file paths to job definitions
+    :type file_paths: `list`[`str`]
+    :param harness_config: Config for the test harness
+    :type harness_config: :class:`ProtocolVerifierConfig`
+    :param file_type: The type of file to send job defs from
+    :type file_type: `Literal`["json", "uml"]
+    """
+    match file_type:
+        case "json":
+            send_job_defs_from_json(
+                harness_config.pv_send_job_defs_url, file_paths, harness_config
+            )
+        case "uml":
+            send_job_defs_from_uml(
+                harness_config.pv_send_job_defs_url, file_paths, harness_config
+            )
+        case _:
+            raise ValueError(f"Invalid file type: {file_type}")
+
+
+def send_job_defs(
+    url: str, job_defs: list[str],
+    file_paths: list[str],
+    harness_config: ProtocolVerifierConfig
+) -> None:
+    """Method to send job defs from a list of job defs and file paths to an url
+    with :class:`ProtocolVerifierConfig`
+
+    :param url: The url to send the request for uploading job definitions
+    :type url: `str`
+    :param job_defs: A list of job definitions as strings
+    :type job_defs: `list`[`str`]
+    :param file_paths: A list of file paths to job definitions
+    :type file_paths: `list`[`str`]
     :param harness_config: Config for the test harness
     :type harness_config: :class:`ProtocolVerifierConfig`
     """
-    job_defs = get_job_defs_from_uml_files(uml_file_paths)
     file_io_file_name_tuples = [
         create_file_io_file_name_tuple_with_file_path(
-            file_path.replace(".puml", ".json"), file_string
+            file_path, job_def
         )
-        for file_path, file_string in zip(uml_file_paths, job_defs)
+        for job_def, file_path in zip(job_defs, file_paths)
     ]
     send_job_defs_from_file_io_file_name_tuples(
         file_io_file_name_tuples=file_io_file_name_tuples,
@@ -38,6 +72,59 @@ def send_job_defs_from_uml(
     )
 
 
+def get_job_defs_from_jsons(json_file_paths: list[str]) -> list[str]:
+    """Method to get job defs from a list of json file paths
+
+    :param json_file_paths: A list of file paths to json file job definitions
+    :type json_file_paths: `list`[`str`]
+    :return: A list of job definitions as strings
+    :rtype: `list`[`str`]
+    """
+    job_defs: list[str] = []
+    for json_file_path in json_file_paths:
+        with open(json_file_path, "r") as json_file:
+            job_defs.append(json_file.read())
+    return job_defs
+
+
+def send_job_defs_from_json(
+    url: str, json_file_paths: list[str],
+    harness_config: ProtocolVerifierConfig
+) -> None:
+    """Method to send job defs from a list of json file paths to an url with
+    :class:`ProtocolVerifierConfig`
+
+    :param url: The url to send the request for uploading job definitions
+    :type url: `str`
+    :param json_file_paths: A list of file paths to json file job definitions
+    :type json_file_paths: `list`[`str`]
+    :param harness_config: Config for the test harness
+    :type harness_config: :class:`ProtocolVerifierConfig`
+    """
+    job_defs = get_job_defs_from_jsons(json_file_paths)
+    send_job_defs(url, job_defs, json_file_paths, harness_config)
+
+
+def send_job_defs_from_uml(
+    url: str, uml_file_paths: list[str], harness_config: ProtocolVerifierConfig
+) -> None:
+    """Method to send job defs from a list of uml file paths to an url with
+    :class:`ProtocolVerifierConfig`
+
+    :param url: The url to send the request for uploading job definitions
+    :type url: `str`
+    :param uml_file_paths: A list of filepaths to uml file job definitions
+    :type uml_file_paths: `list`[`str`]
+    :param harness_config: Config for the test harness
+    :type harness_config: :class:`ProtocolVerifierConfig`
+    """
+    job_defs = get_job_defs_from_uml_files(uml_file_paths)
+    converted_file_paths = [
+        file_path.replace(".puml", ".json") for file_path in uml_file_paths
+    ]
+    send_job_defs(url, job_defs, converted_file_paths, harness_config)
+
+
 def send_job_defs_from_file_io_file_name_tuples(
     file_io_file_name_tuples: list[tuple[BytesIO, str]],
     url: str,

test_harness/protocol_verifier/tests/test_files/test_uml_1_jobdef.json

@@ -0,0 +1,25 @@
+{
+    "JobDefinitionName": "test_uml_1",
+    "Events": [
+        {
+            "EventName": "A",
+            "OccurrenceId": 0,
+            "SequenceName": "test_uml_1",
+            "Application": "default_application_name",
+            "SequenceStart": true
+        },
+        {
+            "EventName": "B",
+            "OccurrenceId": 0,
+            "SequenceName": "test_uml_1",
+            "Application": "default_application_name",
+            "SequenceEnd": true,
+            "PreviousEvents": [
+                {
+                    "PreviousEventName": "A",
+                    "PreviousOccurrenceId": 0
+                }
+            ]
+        }
+    ]
+}

test_harness/protocol_verifier/tests/test_protocol_verifier__init__.py

@@ -108,12 +108,16 @@
 uuid4hex = re.compile("[0-9a-f]{12}4[0-9a-f]{3}[89ab][0-9a-f]{15}\\Z", re.I)
 
 
+@pytest.mark.parametrize("jobdef_type", ["json", "uml"])
 @responses.activate
-def test_puml_files_test() -> None:
+def test_puml_files_test(jobdef_type: Literal["json", "uml"]) -> None:
     """Tests method `puml_test_files`"""
     harness_config = ProtocolVerifierConfig(config_parser)
     test_config = TestConfig()
-    test_config.parse_from_dict({"event_gen_options": {"invalid": False}})
+    test_config.parse_from_dict({
+        "event_gen_options": {"invalid": False},
+        "jobdef_type": jobdef_type,
+    })
     with mock_pv_http_interface(harness_config):
         puml_files_test(
             puml_file_paths=[test_file_path],