tests: Add report, restructure

Add html report Restructure tests, markers Signed-off-by: Jorgen Kvalvaag <[email protected]>
hello-nrfcloud · Nov 26, 2024 · 89aa9f9 · 89aa9f9
1 parent 7c9ef06
commit 89aa9f9
Show file tree

Hide file tree

Showing 13 changed files with 163 additions and 266 deletions.
diff --git a/.github/workflows/on_target.yml b/.github/workflows/on_target.yml
@@ -3,44 +3,23 @@ name: Target tests
 on:
   workflow_call:
     inputs:
+      artifact_run_id:
+        description: The run ID of the workflow to fetch artifacts from
+        type: string
+        required: true
       artifact_fw_version:
+        description: The firmware version found under this run_id
         type: string
         required: true
-      artifact_run_id:
+      pytest_marker:
         type: string
         required: true
-      run_log_check_tests:
-        type: boolean
-        required: false
-        default: true
-      run_fota_tests:
-        type: boolean
-        required: false
-        default: true
-      run_fullmfwfota_test:
-        type: boolean
-        required: false
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: false
-        default: true
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: false
-        default: true
-      run_wifi_location_tests:
-        type: boolean
-        required: false
-        default: true
-      run_modem_trace_tests:
-        type: boolean
-        required: false
-        default: true
-      run_ppk_tests:
-        type: boolean
+        default: -m "not slow"
+      pytest_path:
+        type: string
         required: false
-        default: false
+        default: tests
+
   workflow_dispatch:
     inputs:
       artifact_fw_version:
@@ -51,38 +30,16 @@ on:
         description: The run ID of the workflow to fetch artifacts from
         type: string
         required: true
-      run_log_check_tests:
-        type: boolean
-        required: true
-        default: true
-      run_fota_tests:
-        type: boolean
-        required: true
-        default: false
-      run_fullmfwfota_test:
-        type: boolean
-        required: true
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: true
-        default: false
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: true
-        default: false
-      run_wifi_location_tests:
-        type: boolean
-        required: true
-        default: false
-      run_modem_trace_tests:
-        type: boolean
-        required: true
-        default: false
-      run_ppk_tests:
-        type: boolean
+      pytest_marker:
+        description: The pytest marker to run
+        type: string
         required: true
-        default: false
+        default: -m "not slow"
+      pytest_path:
+        description: Select test execution path
+        type: string
+        required: false
+        default: tests
 
 jobs:
   target_test:
@@ -135,107 +92,27 @@ jobs:
               --software-version ${{ inputs.artifact_fw_version }} \
               hello.nrfcloud.com-${{ inputs.artifact_fw_version }}-thingy91x-nrf91.elf
 
-      - name: Run UART tests
-        if: ${{ inputs.run_log_check_tests }}
+      - name: Tests
         working-directory: thingy91x-oob/tests/on_target
         run: |
           mkdir -p results
-          pytest -s -v -m "dut1 and uart" \
-            --junit-xml=results/test-results-uart.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          LOG_FILENAME: oob_uart_test_log
-
-      - name: Run FOTA tests (standard)
-        if: ${{ inputs.run_fota_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fota" \
-            --junit-xml=results/test-results-fota.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          IMEI: ${{ secrets.IMEI_DUT_1 }}
-          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fota_test_log
-
-      - name: Run FOTA tests (FULLMFW)
-        if: ${{ inputs.run_fullmfwfota_test }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fullmfw_fota" \
-            --junit-xml=results/test-results-fullmfw-fota.xml \
-            tests
+          pytest -s -v ${{ inputs.pytest_marker }} \
+            --junit-xml=results/test-results-dut1.xml \
+            --html=results/test-results-dut1.html --self-contained-html \
+            ${{ inputs.pytest_path }}
         env:
           SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          IMEI: ${{ secrets.IMEI_DUT_1 }}
-          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fullmfw_fota_test_log
-
-      - name: Run DFU tests
-        if: ${{ inputs.run_dfu_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut2 and dfu" \
-            --junit-xml=results/test-results-dfu.xml \
-            tests
-        env:
           SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
           SEGGER_NRF91: ${{ secrets.SEGGER_DUT_2_NRF91 }}
-          UART_ID: ${{ secrets.UART_DUT_2 }}
-
-          LOG_FILENAME: oob_dfu_test_log
-
-      - name: Run Connectivity Bridge Test
-        if: ${{ inputs.run_connectivity_bridge_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut2 and conn_bridge" \
-            --junit-xml=results/test-results-connectivity-bridge.xml \
-            tests
-        env:
-          SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
-          UART_ID: ${{ secrets.UART_DUT_2 }}
-          LOG_FILENAME: oob_conn_bridge_test_log
-
-      - name: Run Wi-Fi location tests
-        if: ${{ inputs.run_wifi_location_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m "dut1 and wifi" \
-            --junit-xml=results/test-results-wifi-location.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-
-      - name: Run modem trace test
-        if: ${{ inputs.run_modem_trace_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m "dut1 and traces" \
-            --junit-xml=results/test-results-traces-location.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-
-      - name: Run PPK tests
-        if: ${{ inputs.run_ppk_tests }}
-        continue-on-error: true
-        id: ppk_test
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m dut_ppk \
-            --junit-xml=results/test-results-ppk.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_PPK }}
+          IMEI: ${{ secrets.IMEI_DUT_1 }}
+          UART_ID_DUT_2: ${{ secrets.UART_DUT_2 }}
+          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
+          LOG_FILENAME: oob_uart_test_log
+          TEST_REPORT_NAME: OOB Firwmare Test Report
+          DUT1_HW_REVISION: ${{ vars.DUT1_HW_REVISION }}
+          SEGGER_PPK: ${{ secrets.SEGGER_DUT_PPK }}
 
       - name: Commit and Push Badge File to gh-pages Branch
-        if: ${{ inputs.run_ppk_tests }}
         continue-on-error: true
         working-directory: thingy91x-oob
         env:
@@ -251,6 +128,15 @@ jobs:
           fail-on-empty: true
           title: OOB FW Test Results
 
+      - name: Create Report Artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        id: artifact-report
+        with:
+          name: test-report
+          path: |
+            thingy91x-oob/tests/on_target/results/*.html
+
       - name: Push log files to artifacts
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,34 +3,11 @@ name: Build and Test
 on:
   workflow_dispatch:
     inputs:
-      run_fota_tests:
-        type: boolean
+      pytest_marker:
+        type: string
         required: true
-        default: true
-      run_fullmfwfota_test:
-        type: boolean
-        required: true
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: true
-        default: true
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: true
-        default: true
-      run_wifi_location_tests:
-        type: boolean
-        required: true
-        default: true
-      run_modem_trace_tests:
-        type: boolean
-        required: true
-        default: true
-      run_ppk_tests:
-        type: boolean
-        required: true
-        default: false
+        default: -m "not slow"
+
   schedule:
     - cron: "0 0 * * *"
   push:
@@ -58,43 +35,4 @@ jobs:
     with:
       artifact_fw_version: ${{ needs.build.outputs.version }}
       artifact_run_id: ${{ needs.build.outputs.run_id }}
-      run_fota_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_fota_tests == 'true')
-        }}
-      run_fullmfwfota_test: >-
-        ${{
-          github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_fullmfwfota_test == 'true')
-        }}
-      run_dfu_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_dfu_tests == 'true')
-        }}
-      run_connectivity_bridge_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_connectivity_bridge_tests == 'true')
-        }}
-      run_wifi_location_tests: >-
-        ${{
-            github.event_name == 'schedule' ||
-            github.event_name == 'push' ||
-            (github.event_name == 'workflow_dispatch' && github.event.inputs.run_wifi_location_tests == 'true')
-        }}
-      run_modem_trace_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_modem_trace_tests == 'true')
-        }}
-      run_ppk_tests: >-
-        ${{
-            github.event_name == 'schedule' ||
-            (github.event_name == 'workflow_dispatch' && github.event.inputs.run_ppk_tests == 'true')
-        }}
+      pytest_marker: ${{ github.event.inputs.pytest_marker || '' }}
diff --git a/tests/on_target/requirements.txt b/tests/on_target/requirements.txt
@@ -1,4 +1,5 @@
 pytest
+pytest-html
 pyserial
 termcolor
 pyusb