feat(coverage): Enable coverage on any test change (rebased with uv) #338
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evmone Coverage Report | |
| on: | |
| pull_request: | |
| paths: | |
| - 'tests/**' # This triggers the workflow for any changes in the tests folder | |
| - '!tests/prague/**' # This excludes changes in 'tests/prague' | |
| jobs: | |
| evmone-coverage-diff: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Debug GitHub context | |
| run: | | |
| echo "Git reference: ${{ github.ref }}" | |
| echo "Git head ref: ${{ github.head_ref }}" | |
| echo "Git base ref: ${{ github.base_ref }}" | |
| - name: Get all changed python files in tests/ and changes to coverted-ethereum-tests.txt | |
| id: changed-tests | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| # TODO: non-test modules such as __init__.py or spec.py could effect coverage - in this case we should | |
| # fill all applicable tests (i.e., all the test_*.py files in or under the changed module's directory) | |
| include_all_old_new_renamed_files: true | |
| output_renamed_files_as_deleted_and_added: true | |
| files_yaml: | | |
| tests: | |
| - tests/**/test_*.py | |
| - '!tests/prague/**' | |
| converted_tests: | |
| - converted-ethereum-tests.txt | |
| - name: Exit workflow if there are no changed python files | |
| if: steps.changed-tests.outputs.tests_any_changed != 'true' | |
| run: | | |
| echo "No python files were changed in ./tests/ - no action necessary" | |
| exit 0 | |
| - name: Report changed python test moudules | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| run: | | |
| echo "${{ toJson(steps.changed-tests.outputs) }}" | |
| echo "Changed python test modules: ${{ steps.changed-tests.outputs.tests_all_modified_files }}" | |
| - name: Debug GitHub context | |
| run: | | |
| echo "Git reference: ${{ github.ref }}" | |
| echo "Git head ref: ${{ github.head_ref }}" | |
| echo "Git base ref: ${{ github.base_ref }}" | |
| - name: Log in to Docker Hub | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository }} | |
| uses: docker/login-action@v3 | |
| with: | |
| username: winsvega | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Install deps | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| run: | | |
| echo $(pwd) | |
| echo ${{ github.workspace }} | |
| - name: Set up uv | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| uses: ./.github/actions/setup-uv | |
| - name: Set up Python | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| run: uv python install 3.10 | |
| - name: Install EEST | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| run: | | |
| uv sync --no-progress | |
| uv run python --version | |
| # Required to fill .py tests | |
| - name: Build EVMONE EVM | |
| uses: ./.github/actions/build-evm-client/evmone | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| id: evm-builder2 | |
| with: | |
| type: "main" | |
| - name: Checkout ethereum/tests | |
| uses: actions/checkout@v4 | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| with: | |
| repository: ethereum/tests | |
| path: testpath | |
| sparse-checkout: | | |
| GeneralStateTests | |
| EOFTests | |
| - name: Checkout ethereum/legacytests | |
| uses: actions/checkout@v4 | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| with: | |
| repository: ethereum/legacytests | |
| path: legacytestpath | |
| sparse-checkout: | | |
| Cancun/GeneralStateTests | |
| # This command diffs the file and filters in new lines | |
| - name: Parse converted tests from converted-ethereum-tests.txt | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| run: | | |
| echo "New lines introduced in converted-ethereum-tests.txt:" | |
| lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true) | |
| if [ -z "$lines" ]; then | |
| echo "No new lines in converted-ethereum-tests.txt, check updates instead:" | |
| echo "converted_skip=true" >> $GITHUB_ENV | |
| exit 0 | |
| else | |
| echo "converted_skip=false" >> $GITHUB_ENV | |
| fi | |
| files=$(echo "$lines" | grep -oP '(?<=\+).+\.json') | |
| for file in $files; do | |
| echo $file | |
| done | |
| echo "----------------" | |
| echo "Discovered existing json tests that will be BASE files:" | |
| BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| mkdir -p $BASE_TESTS_PATH | |
| for file in $files; do | |
| # Make sure each file exist at least in develop or legacy tests | |
| file_found=0 | |
| file_path=${{ github.workspace }}/testpath/$file | |
| if [ -e "$file_path" ]; then | |
| file_found=1 | |
| cp $file_path $BASE_TESTS_PATH | |
| echo $file_path | |
| fi | |
| # Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet) | |
| if [[ "$file" == *"GeneralStateTests"* ]]; then | |
| file_path=${{ github.workspace }}/legacytestpath/Cancun/$file | |
| base_name=$(basename "$file") | |
| legacy_file_name="legacy_$base_name" | |
| if [ -e "$file_path" ]; then | |
| file_found=1 | |
| cp $file_path $BASE_TESTS_PATH/$legacy_file_name | |
| echo $file_path | |
| fi | |
| fi | |
| if [ $file_found -eq 0 ]; then | |
| echo "Error: Failed to find the test file $file in test repo" | |
| exit 1 | |
| fi | |
| done | |
| # This command diffs the .py scripts introduced by a PR | |
| - name: Parse and fill introduced test sources | |
| if: steps.changed-tests.outputs.tests_any_changed == 'true' | |
| env: | |
| CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }} | |
| run: | | |
| source $GITHUB_ENV | |
| files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n') | |
| # fill new tests | |
| # using `|| true` here because if no tests found, pyspec fill returns error code | |
| mkdir -p fixtures/state_tests | |
| mkdir -p fixtures/eof_tests | |
| uv run fill $files --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1 | |
| (uv run fill $files --fork=PragueEIP7692 --evm-bin evmone-t8n || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1 | |
| if grep -q "FAILURES" filloutput.log; then | |
| echo "Error: failed to generate .py tests." | |
| exit 1 | |
| fi | |
| filesState=$(find fixtures/state_tests -type f -name "*.json") | |
| filesEOF=$(find fixtures/eof_tests -type f -name "*.json") | |
| if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then | |
| echo "Error: No filled JSON fixtures found in fixtures." | |
| exit 1 | |
| fi | |
| # Include basic evm operations into coverage by default | |
| # As when we translate from yul/solidity some dup/push opcodes could become untouched | |
| uv run fill tests/homestead/coverage/test_coverage.py --until=Cancun --evm-bin evmone-t8n | |
| PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS | |
| mkdir -p $PATCH_TEST_PATH | |
| find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \; | |
| find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \; | |
| - name: Parse and fill introduced test sources from before the PR | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && env.converted_skip == 'true' }} | |
| env: | |
| CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_modified_files }} | |
| run: | | |
| echo "--------------------" | |
| echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:" | |
| source $GITHUB_ENV | |
| files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n') | |
| git checkout main | |
| PREV_COMMIT=$(git rev-parse HEAD) | |
| echo "Checkout head $PREV_COMMIT" | |
| # Take only those files that exist in the filesystem (ignore newly created files) | |
| files_fixed=$(echo "$files" | tr ' ' '\n' | while read file; do | |
| if [ -f "$file" ]; then | |
| echo "$file" | |
| fi | |
| done | tr '\n' ' ') | |
| echo "Select files that were changed and exist on the main branch:" | |
| echo $files_fixed | |
| rm -r fixtures | |
| rm filloutput.log | |
| rm filloutputEOF.log | |
| mkdir -p fixtures/state_tests | |
| mkdir -p fixtures/eof_tests | |
| uv run fill $files_fixed --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1 | |
| (uv run fill $files_fixed --fork=PragueEIP7692 --evm-bin evmone-t8n || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1 | |
| if grep -q "FAILURES" filloutput.log; then | |
| echo "Error: failed to generate .py tests from before the PR." | |
| exit 1 | |
| fi | |
| if grep -q "ERROR collecting test session" filloutput.log; then | |
| echo "Error: failed to generate .py tests from before the PR." | |
| exit 1 | |
| fi | |
| filesState=$(find fixtures/state_tests -type f -name "*.json") | |
| filesEOF=$(find fixtures/eof_tests -type f -name "*.json") | |
| BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| mkdir -p $BASE_TEST_PATH | |
| find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \; | |
| find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \; | |
| for file in $BASE_TEST_PATH/*.json; do | |
| if [ -e "$file" ]; then | |
| mv "$file" "${file%.json}_$PREV_COMMIT.json" | |
| fi | |
| done | |
| - name: Print tests that will be covered | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| run: | | |
| echo "Original BASE tests:" | |
| ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| echo "--------------------" | |
| echo "Ported PATCH tests:" | |
| ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS | |
| - name: Run coverage of the BASE tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/BASE_TESTS --outputname=BASE | |
| - name: Run coverage of the PATCH tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/PATCH_TESTS --outputname=PATCH | |
| - name: Run coverage DIFF of the PATCH tests compared to BASE tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov | |
| - name: Chmod coverage results | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| run: | | |
| user=$(whoami) | |
| sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage | |
| - name: Upload coverage results | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| with: | |
| name: coverage-diff-native | |
| path: ${{ github.workspace }}/evmtest_coverage/coverage | |
| - name: Verify coverage results | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /check.sh |