From 931f312cd3f5780104d6a834f4fa9383f5b8bc6d Mon Sep 17 00:00:00 2001 From: Christian Freitas Date: Thu, 16 Feb 2023 19:52:47 +0000 Subject: [PATCH 01/80] Update cromwell version from 85 to 86 --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index ca782b990e2..cfdd3352dc2 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -6,7 +6,7 @@ import com.github.sbt.git.SbtGit object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "85" + val cromwellVersion = "86" /** * Returns true if this project should be considered a snapshot. From 8aa0dd560dd785b201e6201f592d6f24f8175dfa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 21 Feb 2023 11:32:25 -0500 Subject: [PATCH 02/80] WX-905 (#7012) Co-authored-by: Thomas Wiseman Co-authored-by: Tom Wiseman Co-authored-by: Janet Gainer-Dewar --- .github/workflows/cromwell_unit_tests.yml | 1 + .github/workflows/integration_tests.yml | 83 +++++++++++++++++++++++ .gitignore | 6 ++ src/ci/bin/test.inc.sh | 39 ++++++++++- 4 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/integration_tests.yml diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index d0927f8b954..797f38efd96 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -9,6 +9,7 @@ run-name: ${{ github.actor }} running Cromwell sbt unit tests. on: workflow_dispatch: #Manual trigger from GitHub UI push: + merge_group: permissions: contents: read diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 00000000000..489e9e731a9 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,83 @@ +name: 'Integration Tests' + +#This github action runs all of Cromwell's integration tests. + +#This is what shows up in the github workflows page as the title. +run-name: ${{ github.actor }} Integration Testing. + +#What will trigger the workflow to run. +on: + workflow_dispatch: #Manual trigger from GitHub UI + push: #git push + merge_group: + +permissions: + contents: read + +jobs: + integration-tests: + strategy: + fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. + #Each entry below is a single integration test that lives in /src/ci/bin/. + #Each will be launched on its own runner so they can occur in parallel. + #Friendly names are displayed on the Github UI and aren't used anywhere else. + matrix: + include: + - build_type: centaurPapiV2beta + build_mysql: 5.7 + friendly_name: Centaur Papi V2 Beta with MySQL 5.7 + - build_type: dbms + friendly_name: DBMS + - build_type: centaurTes + build_mysql: 5.7 + friendly_name: Centaur TES with MySQL 5.7 + - build_type: centaurLocal + build_mysql: 5.7 + friendly_name: Centaur Local with MySQL 5.7 + - build_type: checkPublish + friendly_name: Check Publish + - build_type: centaurAws + build_mysql: 5.7 + friendly_name: Centaur AWS with MySQL 5.7 + - build_type: centaurDummy + build_mysql: 5.7 + friendly_name: Centaur Dummy with MySQL 5.7 + - build_type: centaurHoricromtalPapiV2beta + build_mysql: 5.7 + friendly_name: Centaur Horicromtal PapiV2 Beta with MySQL 5.7 + - build_type: horicromtalDeadlock + friendly_name: Horicromtal Deadlock + - build_type: singleWorkflowRunner + friendly_name: Single Workflow Runner + - build_type: centaurLocal + build_mariadb: 10.3 + friendly_name: Centaur Local with MariaDB 10.3 + - build_type: centaurLocal + build_postgresql: 11.3 + friendly_name: Centaur Local with PostgreSQL 11.3 + - build_type: centaurEngineUpgradeLocal + build_mysql: 5.7 + friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 + name: ${{ matrix.friendly_name }} + env: + BUILD_NAME: ${{ matrix.build_type }} + BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable + BUILD_MYSQL: ${{ matrix.build_mysql }} + BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} + BUILD_MARIADB: ${{ matrix.build_mariadb }} + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + #This script bascially just looks up another script to run, assuming that the other script's filename is: + #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. + - name: Run Integration Test + run: | + set -e + echo Running test.sh + ./src/ci/bin/test.sh diff --git a/.gitignore b/.gitignore index 571a12c5873..a80b4a21713 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,12 @@ tags target /site +#from running integration tests locally +actual.json +console_output.txt +expected.json +run_mode_metadata.json + # custom config cromwell-executions cromwell-test-executions diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 95820513dfe..a5d9d4fe509 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -83,6 +83,7 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER_TRAVIS="travis" CROMWELL_BUILD_PROVIDER_JENKINS="jenkins" CROMWELL_BUILD_PROVIDER_CIRCLE="circle" + CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" if [[ "${TRAVIS-false}" == "true" ]]; then @@ -91,6 +92,8 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_JENKINS}" elif [[ "${CIRCLECI-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_CIRCLE}" + elif [[ "${GITHUB_ACTIONS-false}" == "true" ]]; then + CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_GITHUB}" else CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_UNKNOWN}" fi @@ -300,6 +303,21 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_RUN_TESTS=true fi ;; + "${CROMWELL_BUILD_PROVIDER_GITHUB}") + CROMWELL_BUILD_IS_CI=true + CROMWELL_BUILD_IS_SECURE=true + CROMWELL_BUILD_TYPE="${BUILD_TYPE}" + CROMWELL_BUILD_BRANCH="${GITHUB_REF_NAME}" + CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}" + CROMWELL_BUILD_TAG="" + CROMWELL_BUILD_NUMBER="${GITHUB_RUN_ID}" + CROMWELL_BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + CROMWELL_BUILD_GIT_USER_EMAIL="" + CROMWELL_BUILD_GIT_USER_NAME="${GITHUB_ACTOR}" + CROMWELL_BUILD_HEARTBEAT_PATTERN="…" + CROMWELL_BUILD_GENERATE_COVERAGE=true + CROMWELL_BUILD_RUN_TESTS=true + ;; *) CROMWELL_BUILD_IS_CI=false CROMWELL_BUILD_IS_SECURE=true @@ -476,7 +494,8 @@ cromwell::private::create_database_variables() { case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") + "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ + "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_MARIADB_HOSTNAME="localhost" CROMWELL_BUILD_MARIADB_PORT="23306" CROMWELL_BUILD_MARIADB_DOCKER_TAG="${BUILD_MARIADB-}" @@ -633,7 +652,8 @@ cromwell::private::create_centaur_variables() { # Pick **one** of the databases to run Centaur against case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") + "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ + "${CROMWELL_BUILD_PROVIDER_GITHUB}") if [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$" @@ -1436,6 +1456,21 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_pyenv_python_latest cromwell::private::start_docker_databases ;; + "${CROMWELL_BUILD_PROVIDER_GITHUB}") + # Try to login to vault, and if successful then use vault creds to login to docker. + # For those committers with vault access this avoids pull rate limits reported in BT-143. + cromwell::private::install_vault + cromwell::private::login_vault + cromwell::private::login_docker + #Note: Unlike with other CI providers, we are using Github Actions to install Java and sbt for us. + #This is automatically handled in the set_up_cromwell Github Action, which can be found in + #[cromwell root]/.github/set_up_cromwell_aciton. + cromwell::private::install_docker_compose + cromwell::private::delete_boto_config + cromwell::private::delete_sbt_boot + cromwell::private::upgrade_pip + cromwell::private::start_docker_databases + ;; "${CROMWELL_BUILD_PROVIDER_JENKINS}"|\ *) ;; From c3344cb4dd997b75dd6648b7f38ec14d572c6b6b Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 21 Feb 2023 16:04:06 -0500 Subject: [PATCH 03/80] WX-719 modernize `cromwell-publish` image (#7013) --- publish/Dockerfile | 5 ++--- publish/docker-build.sh | 10 ++++++++-- publish/docker-setup.sh | 26 +++++++++++--------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/publish/Dockerfile b/publish/Dockerfile index 7b276e50490..16dfcff30a7 100644 --- a/publish/Dockerfile +++ b/publish/Dockerfile @@ -1,6 +1,5 @@ -# Alternatively instead of `FROM linuxbrew/brew` we could run all of the steps used to install brew in docker-setup.sh -# https://github.com/Homebrew/brew/blob/0ff2afdfa8c5943a0e55d9bfe3cdb5d11da8342a/Dockerfile -FROM linuxbrew/brew +# Latest stable release including non-LTS +FROM ubuntu:rolling WORKDIR /cromwell-publish/ COPY docker-setup.sh git-setup.sh ./ diff --git a/publish/docker-build.sh b/publish/docker-build.sh index 4630a066dcd..460e62f4430 100755 --- a/publish/docker-build.sh +++ b/publish/docker-build.sh @@ -4,6 +4,12 @@ set -euo pipefail +export DOCKER_CLI_EXPERIMENTAL=enabled + +docker buildx rm cromwell-multi-arch-builder || true +docker buildx create --use --name cromwell-multi-arch-builder + build_root="$( dirname "${BASH_SOURCE[0]}" )" -docker build "${build_root}" -t broadinstitute/cromwell-publish -docker push broadinstitute/cromwell-publish +docker buildx build "${build_root}" --platform linux/amd64,linux/arm64 -t broadinstitute/cromwell-publish:latest --push + +docker buildx rm cromwell-multi-arch-builder diff --git a/publish/docker-setup.sh b/publish/docker-setup.sh index 1ef181bae19..613d251df96 100755 --- a/publish/docker-setup.sh +++ b/publish/docker-setup.sh @@ -4,36 +4,32 @@ set -eou pipefail -apt-get update -apt-get install \ +apt update +apt install \ apt-transport-https \ curl \ git \ gnupg \ wget \ + ca-certificates \ + unzip \ + zip \ -y --no-install-recommends -# setup install for adoptopenjdk -# https://adoptopenjdk.net/installation.html#linux-pkg-deb -wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - -echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $( - grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 - ) main" | - tee /etc/apt/sources.list.d/adoptopenjdk.list +mkdir -p /etc/apt/keyrings +wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc +echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list +apt update +apt install -y temurin-11-jdk # Install jq 1.6 to ensure --rawfile is supported curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/bin/jq chmod +x /usr/bin/jq -apt-get update -apt-get install \ - adoptopenjdk-11-hotspot \ - -y --no-install-recommends - # sbt launcher non-deb package installation instructions adapted from # - https://github.com/sbt/sbt/releases/tag/v1.4.9 # - https://github.com/broadinstitute/scala-baseimage/pull/4/files -curl --location --fail --silent --show-error "https://github.com/sbt/sbt/releases/download/v1.5.5/sbt-1.5.5.tgz" | +curl --location --fail --silent --show-error "https://github.com/sbt/sbt/releases/download/v1.8.2/sbt-1.8.2.tgz" | tar zxf - -C /usr/share update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 From f5a9e054ff510ddb5c6b2482372ba8c0a7c1fcdb Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 21 Feb 2023 16:40:18 -0500 Subject: [PATCH 04/80] WX-930 Add `CODEOWNERS` file (#7015) --- CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000000..34ece8d7792 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,5 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @broadinstitute/dsp-batch will be requested for +# review when someone opens a pull request. +* @broadinstitute/dsp-batch From cb4d8fac492b7498135cbd7d01581af0e175c86a Mon Sep 17 00:00:00 2001 From: Stephen Fleming Date: Tue, 21 Feb 2023 17:38:05 -0500 Subject: [PATCH 05/80] Update FiveMinuteIntro: Java 11 requirement (#6830) Co-authored-by: Adam Nichols --- docs/Releases.md | 5 +++-- docs/tutorials/FiveMinuteIntro.md | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/Releases.md b/docs/Releases.md index 1a446c90bc2..9699d7d44bc 100644 --- a/docs/Releases.md +++ b/docs/Releases.md @@ -6,8 +6,9 @@ You are strongly encouraged to use the latest release of Cromwell whenever possi Cromwell is distributed as a conda package on [conda-forge](https://conda-forge.org/). These instructions need to be followed for [installing the miniconda distribution](https://docs.conda.io/en/latest/miniconda.html) and [activating the conda-forge channel](https://conda-forge.org/#about). After this Cromwell can be installed in the -base environment with `conda install cromwell` or a separate environment for Cromwell can be created with -`conda create -n cromwell cromwell`. If you are using Cromwell for bioinformatics workflows, you might like to take +base environment with `conda install -c conda-forge cromwell` or a separate environment for Cromwell can be created with +`conda create -n cromwell cromwell` (be sure to activate the conda-forge channel first). +If you are using Cromwell for bioinformatics workflows, you might like to take a look at [bioconda](http://bioconda.github.io) as well. The conda installation of Cromwell comes with a wrapper that locates the jar for you and allows for running Cromwell or Womtool with a `cromwell run`, `womtool validate` or other command. Conda also installs the required Java dependency diff --git a/docs/tutorials/FiveMinuteIntro.md b/docs/tutorials/FiveMinuteIntro.md index 2e203354bee..1c16a56c606 100644 --- a/docs/tutorials/FiveMinuteIntro.md +++ b/docs/tutorials/FiveMinuteIntro.md @@ -4,9 +4,11 @@ * A Unix-based operating system (yes, that includes Mac!) * A Java 11 runtime environment - * You can see what you have by running `$ java -version` on a terminal. - * If not, you can download Java [here](https://adoptopenjdk.net/). - * You might need to update the `export JAVA_HOME` in your bash profile to point to your JAVA install location. + * You can see what you have by running `$ java -version` on a terminal. + * If not, consider installing via conda or brew [as explained here](../Releases.md). + * We recommend [SDKMAN](https://sdkman.io/install) to install the latest 11 build of [Temurin](https://adoptium.net/temurin/releases/?version=11) + * `sdk install java 11.0.16-tem` as of the time of this writing + * You might need to update the `export JAVA_HOME` in your bash profile to point to your JAVA install location. * A sense of adventure! ### Goals From 0854d6c79886ea6fa2f4add6064eef139e54a94f Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Wed, 22 Feb 2023 09:35:16 -0500 Subject: [PATCH 06/80] WX-952 Fix Cromwell version update in Helm chart (#7014) --- .github/workflows/chart_update_on_merge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index aa7a2c9c9ad..0eb8facd927 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -76,8 +76,8 @@ jobs: cd cromwhelm git checkout main ls -la - sed -i "s/appVersion.*/appVersion: \"$CROMWELL_VERSION\"/" cromwell-helm/Chart.yaml - sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_VERSION/" cromwell-helm/templates/cromwell.yaml + sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" terra-batch-libchart/values.yaml + git diff git config --global user.name "broadbot" git config --global user.email "broadbot@broadinstitute.org" From 65e122368a8d14bc24e7dc4cf5dcd850be7d757f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 28 Feb 2023 11:49:38 -0500 Subject: [PATCH 07/80] Reference Disk Manifest Builder App Test (#7017) --- .github/workflows/integration_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 489e9e731a9..aec72777d78 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -58,6 +58,8 @@ jobs: - build_type: centaurEngineUpgradeLocal build_mysql: 5.7 friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 + - build_type: referenceDiskManifestBuilderApp + friendly_name: Reference Disk Manifest Builder App name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} @@ -77,7 +79,13 @@ jobs: #This script bascially just looks up another script to run, assuming that the other script's filename is: #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. - name: Run Integration Test + shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' #See comment below run: | set -e echo Running test.sh ./src/ci/bin/test.sh + #The "shell: ..."" line is a way to force the Github Action Runner to use a bash shell that thinks it has a TTY. + #The issue and solution are described here: https://github.com/actions/runner/issues/241#issuecomment-842566950 + #This is only needed for ReferenceDiskManifestBuilderApp test. + #This test uses fancy colors in the output, which likely causes the problem. + #See WX-938. From 514a5469734587bb096df6b482b94fc031b77607 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:43:17 -0500 Subject: [PATCH 08/80] [WM-1696] Update Womtool (#7019) Co-authored-by: Tom Wiseman --- docs/api/RESTAPI.md | 12 +++++++++++- engine/src/main/resources/swagger/cromwell.yaml | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md index ffb2af329de..7a0585ca08e 100644 --- a/docs/api/RESTAPI.md +++ b/docs/api/RESTAPI.md @@ -1,5 +1,5 @@ + + + + + + + + SELECT count(1) + FROM pg_roles + where '${sharedCromwellDbRole}' != '' and pg_roles.rolname = '${sharedCromwellDbRole}'; + + + + ALTER TABLE "CUSTOM_LABEL_ENTRY" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "METADATA_ENTRY" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "SUMMARY_QUEUE_ENTRY" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "SUMMARY_STATUS_ENTRY" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "WORKFLOW_METADATA_SUMMARY_ENTRY" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "sqlmetadatadatabasechangelog" OWNER TO ${sharedCromwellDbRole}; + ALTER TABLE "sqlmetadatadatabasechangeloglock" OWNER TO ${sharedCromwellDbRole}; + + + + diff --git a/database/migration/src/main/resources/sql_metadata_changelog.xml b/database/migration/src/main/resources/sql_metadata_changelog.xml index 1c5b0837a89..0989ec2199a 100644 --- a/database/migration/src/main/resources/sql_metadata_changelog.xml +++ b/database/migration/src/main/resources/sql_metadata_changelog.xml @@ -19,5 +19,12 @@ + + + From a83ec6e76a4cedd0a8f29e29a41940bb6064140f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 29 Sep 2023 16:33:52 -0400 Subject: [PATCH 79/80] [WX-1234] Update CHANGELOG.md (#7227) Merging past CI since this is doc only. --- CHANGELOG.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76746d972c8..a581852c02e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,19 +2,17 @@ ## 86 Release Notes -### HTTPFilesystem Improvements - -WDL `size` engine function now works for HTTP files. - -### Azure ApplicationInsights Logging Support -Cromwell can now send logs to Azure Application Insights. To enable, set environment -variable `APPLICATIONINSIGHTS_INSTRUMENTATIONKEY` to your account's key. [See here for information.](https://learn.microsoft.com/en-us/azure/azure-monitor/app/sdk-connection-string) +### GCP Batch +Cromwell now supports the GCP Batch backend for running workflows. See `Backend` in [ReadTheDocs](https://cromwell.readthedocs.io/en/stable/) for more information. ### Workflow Completion Callback +Cromwell can be configured to send a POST request to a specified URL when a workflow completes. The request body includes the workflow ID, terminal state, +and (if applicable) final outputs or error message. See `WorkflowCallback` in [ReadTheDocs](https://cromwell.readthedocs.io/en/stable/) for more information. -Cromwell can be configured to send a POST request to a specified URL when a workflow completes. The request body -includes the workflow id, terminal state, and (if applicable) final outputs or error message. -See `WorkflowCallback` in [ReadTheDocs](https://cromwell.readthedocs.io/en/stable/) for more information. +### Other Improvements +* Cromwell will now parallelize the downloads of DRS files that resolve to signed URLs. This significantly reduces the time localization takes in certain situations. +* WDL size engine function now works for HTTP files +* Improved Cromwell's handling of docker manifests. Additional logging information is emitted, and Cromwell will fall back to using OCI manifests if it encounters an error with a Docker Image Manifest V2. ## 85 Release Notes From fc9e3cc9ceb081396d3b17027b11ddef14dced93 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 2 Oct 2023 10:35:30 -0400 Subject: [PATCH 80/80] WM-2252: Configurable metadata write role (#7225) Co-authored-by: dvoet --- .../database/slick/MetadataSlickDatabase.scala | 10 +++++++--- .../finalization/WorkflowCallbackActorSpec.scala | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index 0a33e4b6b72..eb87f88d101 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -1,7 +1,6 @@ package cromwell.database.slick import java.sql.Timestamp - import cats.syntax.functor._ import cats.instances.future._ import com.typesafe.config.{Config, ConfigFactory} @@ -10,6 +9,7 @@ import cromwell.database.sql.MetadataSqlDatabase import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.joins.{CallOrWorkflowQuery, CallQuery, MetadataJobQueryValue, WorkflowQuery} import cromwell.database.sql.tables.{CustomLabelEntry, InformationSchemaEntry, MetadataEntry, WorkflowMetadataSummaryEntry} +import net.ceedubs.ficus.Ficus._ import slick.basic.DatabasePublisher import slick.jdbc.{ResultSetConcurrency, ResultSetType} @@ -60,6 +60,8 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) import dataAccess.driver.api._ import MetadataSlickDatabase._ + lazy val pgLargeObjectWriteRole: Option[String] = originalDatabaseConfig.as[Option[String]]("pgLargeObjectWriteRole") + override def existsMetadataEntries()(implicit ec: ExecutionContext): Future[Boolean] = { val action = dataAccess.metadataEntriesExists.result runTransaction(action) @@ -87,6 +89,8 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) rootWorkflowIdKey, labelMetadataKey) + val roleSet = pgLargeObjectWriteRole.map(role => sqlu"""SET ROLE TO "#$role"""") + // These entries also require a write to the summary queue. def writeSummarizable(): Future[Unit] = if (partitioned.summarizableMetadata.isEmpty) Future.successful(()) else { val batchesToWrite = partitioned.summarizableMetadata.grouped(insertBatchSize).toList @@ -94,13 +98,13 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) val insertMetadata = dataAccess.metadataEntryIdsAutoInc ++= batch insertMetadata.flatMap(ids => writeSummaryQueueEntries(ids)) } - runTransaction(DBIO.sequence(insertActions)).void + runTransaction(DBIO.sequence(roleSet ++ insertActions)).void } // Non-summarizable metadata that only needs to go to the metadata table can be written much more efficiently // than summarizable metadata. def writeNonSummarizable(): Future[Unit] = if (partitioned.nonSummarizableMetadata.isEmpty) Future.successful(()) else { - val action = DBIO.sequence(partitioned.nonSummarizableMetadata.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _)) + val action = DBIO.sequence(roleSet ++ partitioned.nonSummarizableMetadata.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _)) runLobAction(action).void } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala index c469d1dcaa2..97479d348ae 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala @@ -35,7 +35,7 @@ class WorkflowCallbackActorSpec private val serviceRegistryActor = TestProbe("testServiceRegistryActor") private val deathWatch = TestProbe("deathWatch") private val mockUri = new URI("http://example.com") - private val basicConfig = WorkflowCallbackConfig.empty.copy(enabled = true) + private val basicConfig = WorkflowCallbackConfig.empty.copy(enabled = true).copy(retryBackoff = SimpleExponentialBackoff(100.millis, 200.millis, 1.1)) private val basicOutputs = WomMocks.mockOutputExpectations(List("foo" -> WomString("bar")).toMap) private val httpSuccess = Future.successful(HttpResponse.apply(StatusCodes.OK))