-
Notifications
You must be signed in to change notification settings - Fork 0
192 lines (152 loc) · 7.33 KB
/
end-to-end-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
name: End-to-End Test
on:
pull_request:
types:
- opened
- synchronize
branches:
'main'
jobs:
build-and-test:
if: |
github.repository == 'xtuml/erebus'
&& github.actor != 'dependabot[bot]'
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install libraries
run: sudo apt-get install -y libxml2-utils
- name: Setup test environment
run: |
git clone https://github.com/xtuml/munin.git
cd munin
git checkout tags/1.3.1-midstage3
# Copy over necessary files to protocol verifier to get tests to pass
cd ..
cp ./end_to_end_test_files/log-pv-files.properties ./munin/deploy/config/log-pv-files.properties
cp ./end_to_end_test_files/log-pv-kafka.properties ./munin/deploy/config/log-pv-kafka.properties
cp ./end_to_end_test_files/docker-compose.prod.yml ./munin/deploy/docker-compose.prod.yml
docker compose -f ./munin/deploy/docker-compose.prod.yml up -d
# copy over config file that allows tests to run within Github Actions
mkdir -p config
cp ./end_to_end_test_files/config.config ./config/config.config
# Start the test harness
docker compose -f ./docker-compose-end-to-end-test.yml up -d
sleep 10
echo "Testing that no tests are running"
curl 127.0.0.1:8800/isTestRunning | grep 'false'
- name: Run performance test
run: |
echo "Running a performance test"
timeout 1m ./scripts/end-to-end-curl-commands-performance-test.sh
# this should return true as tests are running
echo "Testing that tests are running"
curl 127.0.0.1:8800/isTestRunning | grep 'true'
# It takes this long to get any meaningful output from the test harness
# It's 60 seconds for the test harness to start the job ...
# ... and 10 seconds to run the job ...
# ... and 60 seconds to ensure the test is finished ...
# ... and another 50 seconds for grace time in starting up/waiting for logs/waiting for calculations
echo "Sleeping to give the test harness time to work"
date
sleep 120
date
curl 127.0.0.1:8800/isTestRunning | grep 'false'
docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
rm ./munin/deploy/config/job_definitions/*.json
- name: Check for performance test failures
run: |
# Inspect report output for failures of performance test
failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/performance_test/Report.xml)
# Print the result
echo "Performance test failures: $failures"
# Check if failures is not zero
if [ "$failures" -ne 0 ]; then
echo "There are performance test failures!"
exit 1
else
echo "No test failures."
exit 0
fi
- name: Run performance test job def json
run: |
echo "Running a performance test"
timeout 1m ./scripts/end-to-end-curl-commands-performance-test-jobdefjson.sh
# this should return true as tests are running
echo "Testing that tests are running"
curl 127.0.0.1:8800/isTestRunning | grep 'true'
# It takes this long to get any meaningful output from the test harness
# It's 60 seconds for the test harness to start the job ...
# ... and 10 seconds to run the job ...
# ... and 60 seconds to ensure the test is finished ...
# ... and another 50 seconds for grace time in starting up/waiting for logs/waiting for calculations
echo "Sleeping to give the test harness time to work"
date
sleep 120
date
curl 127.0.0.1:8800/isTestRunning | grep 'false'
docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
rm ./munin/deploy/config/job_definitions/*.json
- name: Check for performance test failures job def json
run: |
# Inspect report output for failures of performance test
failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/performance_test_jobdef/Report.xml)
# Print the result
echo "Performance test failures: $failures"
# Check if failures is not zero
if [ "$failures" -ne 0 ]; then
echo "There are performance test failures!"
exit 1
else
echo "No test failures."
exit 0
fi
- name: Run functional test
run: |
echo "Running a functional test"
timeout 1m ./scripts/end-to-end-curl-commands-functional-test.sh
# this should return true as tests are running
echo "Testing that tests are running"
curl 127.0.0.1:8800/isTestRunning | grep 'true'
# It takes this long to get any meaningful output from the test harness
# It's 60 seconds for the test harness to start the job ...
# ... and 10 seconds to run the job ...
# ... and 60 seconds to ensure the test is finished ...
# ... and another 50 seconds for grace time in starting up/waiting for logs/waiting for calculations
echo "Sleeping to give the test harness time to work"
date
sleep 120
date
curl 127.0.0.1:8800/isTestRunning | grep 'false'
docker compose -f ./docker-compose-end-to-end-test.yml logs test-harness | grep -Po "Test Harness test run completed successfully"
rm ./munin/deploy/config/job_definitions/*.json
- name: Check for functional test failures
run: |
# Inspect report output for failures of functional test
failures=$(xmllint --xpath 'string(//testsuites/@failures)' ./report_output/functional_test/Results.xml)
# These are known failures, so we will discount them
stacked_solution_failures=$(xmllint --xpath 'string(//testsuite[@name="simple_XOR_job.False.StackedSolutions"]/@failures)' ./report_output/functional_test/Results.xml)
# Calculate non stacked solution failures
non_stacked_solution_failures=$(($failures - $stacked_solution_failures))
# Print the result
echo "Known stacked solution failures: $stacked_solution_failures"
echo "Non stacked solution failures: $non_stacked_solution_failures"
# Check if non stacked solution failures is greater than 0
if [ "$non_stacked_solution_failures" -ne 0 ]; then
echo "There are functional test failures!"
exit 1
else
echo "No test failures."
exit 0
fi
- name: Run tear down procedure
run: |
docker compose -f ./munin/deploy/docker-compose.prod.yml down
docker compose -f ./docker-compose-end-to-end-test.yml down