Merge branch 'master' into update-bclconvert

.github/actions/nf-test-action/action.yml

@@ -10,9 +10,9 @@ inputs:
   total_shards:
     description: "Total number of test shards(NOT the total number of matrix jobs)"
     required: true
-  tags:
-    description: "Tags to test (`<tags>[,<tags>...]`)"
-    required: false
+  paths:
+    description: "Test paths"
+    required: true
 
 runs:
   using: "composite"
@@ -72,19 +72,16 @@ runs:
       env:
         SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
-        TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
       run: |
         NFT_WORKDIR=~ \
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \
           --verbose \
           --ci \
-          --changed-since HEAD^ \
           --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
           --filter process,workflow \
-          --follow-dependencies \
-          ${{ env.TAGS }}
+          ${{ inputs.paths }}
 
       # TODO If no test.tap, then make one to spoof?
     - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1

.github/workflows/gpu-tests.yml

@@ -6,21 +6,15 @@ on:
       - "renovate/**" # branches Renovate creates
   pull_request:
     branches: [master]
-    paths:
-      - ".github/workflows/gpu-tests.yml"
-      - "modules/nf-core/parabricks/**"
   merge_group:
     types: [checks_requested]
     branches: [master]
   workflow_dispatch:
     inputs:
       runners:
         description: "Runners to test on"
-        type: choice
-        options:
-          - "ubuntu-latest"
-          - "self-hosted"
-        default: "self-hosted"
+        type: string
+        default: "gpu"
 
 # Cancel if a newer run is started
 concurrency:
@@ -43,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       # Expose detected tags as 'modules' and 'workflows' output variables
-      paths: ${{ steps.outputs.outputs.components }}
+      paths: ${{ steps.list.outputs.components }}
       modules: ${{ steps.outputs.outputs.modules }}
       subworkflows: ${{ steps.outputs.outputs.subworkflows}}
       # Prod for version bumping
@@ -60,11 +54,11 @@ jobs:
 
       - name: List nf-test files
         id: list
-        uses: adamrtalbot/detect-nf-test-changes@6bf6fd9fe0fb63a0362fb0e09de5acb6d055a754 # v0.0.5
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
         with:
           head: ${{ github.sha }}
           base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
-          n_parents: 2
+          n_parents: 0
           tags: "gpu"
 
       - name: Separate modules and subworkflows
@@ -74,21 +68,20 @@ jobs:
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
       - name: debug
         run: |
-          echo ${{ steps.outputs.outputs.components }}
+          echo ${{ steps.list.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
 
   nf-test-gpu:
     runs-on: "gpu"
+    name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}"
     needs: nf-test-changes
-    if: ${{ fromJSON(needs.nf-test-changes.outputs.paths) != '[]' || needs.nf-test-changes.outputs.paths != '' }}
-    name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2]
         profile: [docker_self_hosted, singularity] # conda?
-        tags: ${{ fromJSON(needs.nf-test-changes.outputs.modules) && fromJSON(needs.nf-test-changes.outputs.subworkflows) }}
     env:
       NXF_ANSI_LOG: false
       TOTAL_SHARDS: 2
@@ -108,4 +101,22 @@ jobs:
           profile: ${{ matrix.profile }},gpu
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
-          tags: ${{matrix.tags}},gpu
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test-gpu]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

.github/workflows/lint.yml

@@ -80,7 +80,6 @@ jobs:
           script: |
             return [...new Set(${{ steps.filter.outputs.modules_files }}
               .map(path => path
-                .replace('tests/', '')
                 .replace('modules/nf-core/', '')
                 .split('/')
                 .slice(0, 2)

.github/workflows/nf-test.yml

@@ -35,45 +35,90 @@ env:
   NXF_VER: "24.10.1"
 
 jobs:
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose detected tags as 'modules' and 'workflows' output variables
+      paths: ${{ steps.list.outputs.components }}
+      modules: ${{ steps.outputs.outputs.modules }}
+      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
+      # Prod for version bumping
+    steps:
+      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
+        with:
+          head: ${{ github.sha }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+          n_parents: 0
+          exclude_tags: "gpu"
+
+      - name: Separate modules and subworkflows
+        id: outputs
+        run: |
+          echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+      - name: debug
+        run: |
+          echo ${{ steps.list.outputs.components }}
+          echo ${{ steps.outputs.outputs.modules }}
+          echo ${{ steps.outputs.outputs.subworkflows }}
   nf-test:
     runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
-    # NOTE I think this is the cleanest way to get them organized
-    # process | conda | 1
-    # process | conda | 2
-    # process | conda | 3
-    # process | docker_self_hosted | 1
-    # ...
-    # workflow | singularity | 3
     name: "${{ matrix.profile }} | ${{ matrix.shard }}"
-    # TODO
-    # needs: get-number-of-shards
-    # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }}
+    needs: nf-test-changes
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now
-        # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}"
-        # filter: [process, workflow]
-        profile: [conda, docker_self_hosted, singularity]
         shard: [1, 2, 3, 4, 5]
+        profile: [conda, docker_self_hosted, singularity]
     env:
-      # FIXME Bumping them up to make the transition smooth, then we can throttle them back
+      NXF_ANSI_LOG: false
       TOTAL_SHARDS: 5
-      SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
-      SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+
     steps:
-      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        run: |
-          ls -la ./
-          rm -rf ./* || true
-          rm -rf ./.??* || true
-          ls -la ./
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           fetch-depth: 0
       - name: Run nf-test Action
         uses: ./.github/actions/nf-test-action
+        env:
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
         with:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

modules/nf-core/bismark/align/main.nf

@@ -45,7 +45,7 @@ process BISMARK_ALIGN {
 
         // Check that we have enough memory
         try {
-            def tmem = (task.memory as nextflow.util.MemoryUnit).toBytes()
+            def tmem = (task.memory as MemoryUnit).toBytes()
             def mcore = (tmem / mem_per_multicore) as int
             ccore = Math.min(ccore, mcore)
         } catch (all) {

modules/nf-core/gawk/main.nf

@@ -28,13 +28,19 @@ process GAWK {
     lst_gz    = input.collect{ it.getExtension().endsWith("gz") }
     unzip     = lst_gz.contains(false) ? "" : "find ${input} -exec zcat {} \\; | \\"
     input_cmd = unzip ? "" : "${input}"
+    output_cmd = suffix.endsWith("gz") ? "| gzip" : ""
+
+    input.collect{
+        assert it.name != "${prefix}.${suffix}" : "Input and output names are the same, set prefix in module configuration to disambiguate!"
+    }
 
     """
     ${unzip}
     awk \\
         ${args} \\
         ${program} \\
         ${input_cmd} \\
+        ${output_cmd} \\
         > ${prefix}.${suffix}
 
     cat <<-END_VERSIONS > versions.yml

modules/nf-core/gawk/tests/main.nf.test

@@ -8,10 +8,14 @@ nextflow_process {
     tag "modules_nfcore"
     tag "gawk"
 
-    test("Convert fasta to bed") {
-        config "./nextflow.config"
+    config "./nextflow.config"
 
+    test("Convert fasta to bed") {
         when {
+            params {
+                gawk_suffix = "bed"
+                gawk_args2  = '\'BEGIN {FS="\t"}; {print \$1 FS "0" FS \$2}\''
+            }
             process {
                 """
                 input[0] = [
@@ -32,9 +36,11 @@ nextflow_process {
     }
 
     test("Convert fasta to bed with program file") {
-        config "./nextflow_with_program_file.config"
-
         when {
+            params {
+                gawk_suffix = "bed"
+                gawk_args2  = ""
+            }
             process {
                 """
                 input[0] = [
@@ -55,9 +61,11 @@ nextflow_process {
     }
 
     test("Extract first column from multiple files") {
-        config "./nextflow_with_program_file.config"
-        tag "test"
         when {
+            params {
+                gawk_suffix = "bed"
+                gawk_args2  = ""
+            }
             process {
                 """
                 input[0] = [
@@ -79,9 +87,11 @@ nextflow_process {
     }
 
     test("Unzip files before processing") {
-        config "./nextflow_with_program_file.config"
-
         when {
+            params {
+                gawk_suffix = "bed"
+                gawk_args2  = ""
+            }
             process {
                 """
                 input[0] = [
@@ -101,4 +111,55 @@ nextflow_process {
             )
         }
     }
+
+    test("Compress after processing") {
+        when {
+            params {
+                gawk_suffix = "txt.gz"
+                gawk_args2  = '\'BEGIN {FS="\t"}; {print \$1 FS "0" FS \$2}\''
+            }
+            process {
+                """
+                input[0] = [
+                    [ id:'test' ], // meta map
+                    file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta.fai', checkIfExists: true)
+                ]
+                input[1] = []
+                """
+            }
+        }
+
+        then {
+            assertAll(
+                { assert process.success },
+                { assert snapshot(process.out).match() }
+            )
+        }
+    }
+
+    test("Input and output files are similar") {
+        when {
+            params {
+                gawk_suffix = "txt"
+                gawk_args2  = ""
+            }
+            process {
+                """
+                input[0] = [
+                    [ id:'hello' ], // meta map
+                    [file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true),
+                    file(params.modules_testdata_base_path + 'generic/txt/species_names.txt', checkIfExists: true)]
+                ]
+                input[1] = Channel.of('BEGIN {FS=" "}; {print \$1}').collectFile(name:"program.txt")
+                """
+            }
+        }
+
+        then {
+            assertAll(
+                { assert process.failed },
+                { assert process.errorReport.contains("Input and output names are the same, set prefix in module configuration to disambiguate!") }
+            )
+        }
+    }
 }