Merge branch 'master' into plink_genome

.github/actions/nf-test-action/action.yml

@@ -10,9 +10,9 @@ inputs:
   total_shards:
     description: "Total number of test shards(NOT the total number of matrix jobs)"
     required: true
-  tags:
-    description: "Tags to test (`<tags>[,<tags>...]`)"
-    required: false
+  paths:
+    description: "Test paths"
+    required: true
 
 runs:
   using: "composite"
@@ -72,19 +72,16 @@ runs:
       env:
         SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
-        TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
       run: |
         NFT_WORKDIR=~ \
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \
           --verbose \
           --ci \
-          --changed-since HEAD^ \
           --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
           --filter process,workflow \
-          --follow-dependencies \
-          ${{ env.TAGS }}
+          ${{ inputs.paths }}
 
       # TODO If no test.tap, then make one to spoof?
     - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1

.github/workflows/gpu-tests.yml

@@ -6,21 +6,15 @@ on:
       - "renovate/**" # branches Renovate creates
   pull_request:
     branches: [master]
-    paths:
-      - ".github/workflows/gpu-tests.yml"
-      - "modules/nf-core/parabricks/**"
   merge_group:
     types: [checks_requested]
     branches: [master]
   workflow_dispatch:
     inputs:
       runners:
         description: "Runners to test on"
-        type: choice
-        options:
-          - "ubuntu-latest"
-          - "self-hosted"
-        default: "self-hosted"
+        type: string
+        default: "gpu"
 
 # Cancel if a newer run is started
 concurrency:
@@ -43,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       # Expose detected tags as 'modules' and 'workflows' output variables
-      paths: ${{ steps.outputs.outputs.components }}
+      paths: ${{ steps.list.outputs.components }}
       modules: ${{ steps.outputs.outputs.modules }}
       subworkflows: ${{ steps.outputs.outputs.subworkflows}}
       # Prod for version bumping
@@ -60,11 +54,11 @@ jobs:
 
       - name: List nf-test files
         id: list
-        uses: adamrtalbot/detect-nf-test-changes@6bf6fd9fe0fb63a0362fb0e09de5acb6d055a754 # v0.0.5
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
         with:
           head: ${{ github.sha }}
           base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
-          n_parents: 2
+          n_parents: 0
           tags: "gpu"
 
       - name: Separate modules and subworkflows
@@ -74,21 +68,20 @@ jobs:
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
       - name: debug
         run: |
-          echo ${{ steps.outputs.outputs.components }}
+          echo ${{ steps.list.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
 
   nf-test-gpu:
     runs-on: "gpu"
+    name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}"
     needs: nf-test-changes
-    if: ${{ fromJSON(needs.nf-test-changes.outputs.paths) != '[]' || needs.nf-test-changes.outputs.paths != '' }}
-    name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2]
         profile: [docker_self_hosted, singularity] # conda?
-        tags: ${{ fromJSON(needs.nf-test-changes.outputs.modules) && fromJSON(needs.nf-test-changes.outputs.subworkflows) }}
     env:
       NXF_ANSI_LOG: false
       TOTAL_SHARDS: 2
@@ -108,4 +101,22 @@ jobs:
           profile: ${{ matrix.profile }},gpu
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
-          tags: ${{matrix.tags}},gpu
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test-gpu]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

.github/workflows/nf-test.yml

@@ -35,45 +35,90 @@ env:
   NXF_VER: "24.10.1"
 
 jobs:
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose detected tags as 'modules' and 'workflows' output variables
+      paths: ${{ steps.list.outputs.components }}
+      modules: ${{ steps.outputs.outputs.modules }}
+      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
+      # Prod for version bumping
+    steps:
+      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
+        with:
+          head: ${{ github.sha }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+          n_parents: 0
+          exclude_tags: "gpu"
+
+      - name: Separate modules and subworkflows
+        id: outputs
+        run: |
+          echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+      - name: debug
+        run: |
+          echo ${{ steps.list.outputs.components }}
+          echo ${{ steps.outputs.outputs.modules }}
+          echo ${{ steps.outputs.outputs.subworkflows }}
   nf-test:
     runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
-    # NOTE I think this is the cleanest way to get them organized
-    # process | conda | 1
-    # process | conda | 2
-    # process | conda | 3
-    # process | docker_self_hosted | 1
-    # ...
-    # workflow | singularity | 3
     name: "${{ matrix.profile }} | ${{ matrix.shard }}"
-    # TODO
-    # needs: get-number-of-shards
-    # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }}
+    needs: nf-test-changes
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now
-        # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}"
-        # filter: [process, workflow]
-        profile: [conda, docker_self_hosted, singularity]
         shard: [1, 2, 3, 4, 5]
+        profile: [conda, docker_self_hosted, singularity]
     env:
-      # FIXME Bumping them up to make the transition smooth, then we can throttle them back
+      NXF_ANSI_LOG: false
       TOTAL_SHARDS: 5
-      SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
-      SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+
     steps:
-      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        run: |
-          ls -la ./
-          rm -rf ./* || true
-          rm -rf ./.??* || true
-          ls -la ./
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           fetch-depth: 0
       - name: Run nf-test Action
         uses: ./.github/actions/nf-test-action
+        env:
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
         with:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

modules/nf-core/bismark/align/main.nf

@@ -45,7 +45,7 @@ process BISMARK_ALIGN {
 
         // Check that we have enough memory
         try {
-            def tmem = (task.memory as nextflow.util.MemoryUnit).toBytes()
+            def tmem = (task.memory as MemoryUnit).toBytes()
             def mcore = (tmem / mem_per_multicore) as int
             ccore = Math.min(ccore, mcore)
         } catch (all) {

modules/nf-core/gffread/tests/main.nf.test

@@ -23,6 +23,7 @@ nextflow_process {
                     file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.gff3", checkIfExists: true)
                 ]
                 input[1] = []
+
                 """
             }
         }
@@ -220,4 +221,4 @@ nextflow_process {
 
     }
 
-}
+}

modules/nf-core/hisat2/build/main.nf

@@ -33,7 +33,7 @@ process HISAT2_BUILD {
     def ss = ''
     def exon = ''
     def extract_exons = ''
-    def hisat2_build_memory = params.hisat2_build_memory ? (params.hisat2_build_memory as nextflow.util.MemoryUnit).toGiga() : 0
+    def hisat2_build_memory = params.hisat2_build_memory ? (params.hisat2_build_memory as MemoryUnit).toGiga() : 0
     if (avail_mem >= hisat2_build_memory) {
         log.info "[HISAT2 index build] At least ${hisat2_build_memory} GB available, so using splice sites and exons to build HISAT2 index"
         extract_exons = gtf ? "hisat2_extract_exons.py $gtf > ${gtf.baseName}.exons.txt" : ""

modules/nf-core/muse/call/environment.yml

@@ -0,0 +1,5 @@
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - "bioconda::muse=2.1.2"

modules/nf-core/muse/call/main.nf

@@ -0,0 +1,50 @@
+process MUSE_CALL {
+    tag "$meta.id"
+    label 'process_medium'
+
+    conda "${moduleDir}/environment.yml"
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9f/9f0ebb574ef5eed2a6e034f1b2feea6c252d1ab0c8bc5135a669059aa1f4d2ca/data':
+        'community.wave.seqera.io/library/muse:6637291dcbb0bdb8' }"
+
+    input:
+    tuple val(meta), path(tumor_bam), path(tumor_bai), path(normal_bam), path(normal_bai)
+    tuple val(meta2), path(reference)
+
+    output:
+    tuple val(meta), path("*.MuSE.txt"), emit: txt
+    path "versions.yml"                , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    MuSE \\
+        call \\
+        $args \\
+        -f $reference \\
+        -O ${prefix}  \\
+        -n $task.cpus \\
+        $tumor_bam    \\
+        $normal_bam
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        MuSE: \$( MuSE --version | sed -e "s/MuSE, version //g" )
+    END_VERSIONS
+    """
+
+    stub:
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    touch ${prefix}.MuSE.txt
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        MuSE: \$( MuSE --version | sed -e "s/MuSE, version //g" )
+    END_VERSIONS
+    """
+}