From 4faf200b6ee8b08a4d8ebd26a3393fcfaa998103 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 10:33:21 +0200 Subject: [PATCH 01/10] bump pipeline version also in `.nf-core.yml` --- nf_core/pipelines/bump_version.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index 18aa869328..c6a2f59f33 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -107,6 +107,18 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: ) ], ) + # .nf-core.yml - pipeline version + # update entry: version: 1.0.0dev, but not `nf_core_version`, or `bump_version` + update_file_version( + ".nf-core.yml", + pipeline_obj, + [ + ( + rf"$\s+(version:\s*){re.escape(current_version)}", + rf"\g<1>{new_version}", + ) + ], + ) def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: From 84963ab8d9a3dfff85e9529c347d0e3d0cfd1bb3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 10:34:14 +0200 Subject: [PATCH 02/10] only log and write new contents, if pattern was actually found --- nf_core/pipelines/bump_version.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index c6a2f59f33..fe5730a714 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -206,6 +206,7 @@ def update_file_version(filename: Union[str, Path], pipeline_obj: Pipeline, patt return replacements = [] + updated_version = False for pattern in patterns: found_match = False @@ -229,14 +230,15 @@ def update_file_version(filename: Union[str, Path], pipeline_obj: Pipeline, patt if found_match: content = "\n".join(newcontent) + "\n" + updated_version = True else: log.error(f"Could not find version number in {filename}: `{pattern}`") - - log.info(f"Updated version in '{filename}'") - for replacement in replacements: - stderr.print(f" [red] - {replacement[0].strip()}", highlight=False) - stderr.print(f" [green] + {replacement[1].strip()}", highlight=False) - stderr.print("\n") - - with open(fn, "w") as fh: - fh.write(content) + if updated_version: + log.info(f"Updated version in '{filename}'") + for replacement in replacements: + stderr.print(f" [red] - {replacement[0].strip()}", highlight=False) + stderr.print(f" [green] + {replacement[1].strip()}", highlight=False) + stderr.print("\n") + + with open(fn, "w") as fh: + fh.write(content) From 58a4487c6de5ee4d8c854f363054d77e9a86a5f5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 10:40:01 +0200 Subject: [PATCH 03/10] add some types --- nf_core/pipelines/bump_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index fe5730a714..b5727decc1 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -205,8 +205,8 @@ def update_file_version(filename: Union[str, Path], pipeline_obj: Pipeline, patt log.warning(f"File not found: '{fn}'") return - replacements = [] - updated_version = False + replacements: List[Tuple[str, str]] = [] + updated_version: bool = False for pattern in patterns: found_match = False From 50027db437414b33d7ee5572f64076e7b0113b51 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 13:06:19 +0200 Subject: [PATCH 04/10] rewrite the logic in bump_version.py to handle yml files better --- nf_core/pipelines/bump_version.py | 153 ++++++++++++++++++------------ 1 file changed, 93 insertions(+), 60 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index b5727decc1..55fb82ad20 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -5,7 +5,7 @@ import logging import re from pathlib import Path -from typing import List, Tuple, Union +from typing import List, Optional, Tuple, Union import rich.console @@ -60,6 +60,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: f"/releases/tag/{new_version}", ) ], + yaml_key=["report_comment"], ) if multiqc_current_version != "dev" and multiqc_new_version == "dev": update_file_version( @@ -71,6 +72,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: "/tree/dev", ) ], + yaml_key=["report_comment"], ) if multiqc_current_version == "dev" and multiqc_new_version != "dev": update_file_version( @@ -82,6 +84,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: f"/releases/tag/{multiqc_new_version}", ) ], + yaml_key=["report_comment"], ) update_file_version( Path("assets", "multiqc_config.yml"), @@ -92,6 +95,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: f"/{multiqc_new_version}/", ), ], + yaml_key=["report_comment"], ) # nf-test snap files pipeline_name = pipeline_obj.nf_config.get("manifest.name", "").strip(" '\"") @@ -114,10 +118,12 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: pipeline_obj, [ ( - rf"$\s+(version:\s*){re.escape(current_version)}", - rf"\g<1>{new_version}", + current_version, + new_version, ) ], + required=False, + yaml_key=["template", "version"], ) @@ -159,10 +165,11 @@ def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: # example: # NXF_VER: # - "20.04.0" - rf"- \"{re.escape(current_version)}\"", - f'- "{new_version}"', + current_version, + new_version, ) ], + yaml_key=["jobs", "test", "strategy", "matrix", "NXF_VER"], ) # README.md - Nextflow version badge @@ -183,62 +190,88 @@ def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: ) -def update_file_version(filename: Union[str, Path], pipeline_obj: Pipeline, patterns: List[Tuple[str, str]]) -> None: - """Updates the version number in a requested file. +def update_file_version( + filename: Union[str, Path], + pipeline_obj: Pipeline, + patterns: List[Tuple[str, str]], + required: bool = True, + yaml_key: Optional[List[str]] = None, +) -> None: + fn: Path = pipeline_obj._fp(filename) - Args: - filename (str): File to scan. - pipeline_obj (nf_core.pipelines.lint.PipelineLint): A PipelineLint object that holds information - about the pipeline contents and build files. - pattern (str): Regex pattern to apply. - - Raises: - ValueError, if the version number cannot be found. - """ - # Load the file - fn = pipeline_obj._fp(filename) - content = "" - try: - with open(fn) as fh: - content = fh.read() - except FileNotFoundError: + if not fn.exists(): log.warning(f"File not found: '{fn}'") return - replacements: List[Tuple[str, str]] = [] - updated_version: bool = False - for pattern in patterns: - found_match = False - - newcontent = [] - for line in content.splitlines(): - # Match the pattern - matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) - if matches_pattern: - found_match = True - - # Replace the match - newline = re.sub(pattern[0], pattern[1], line) - newcontent.append(newline) - - # Save for logging - replacements.append((line, newline)) - - # No match, keep line as it is - else: - newcontent.append(line) - - if found_match: - content = "\n".join(newcontent) + "\n" - updated_version = True - else: - log.error(f"Could not find version number in {filename}: `{pattern}`") - if updated_version: - log.info(f"Updated version in '{filename}'") - for replacement in replacements: - stderr.print(f" [red] - {replacement[0].strip()}", highlight=False) - stderr.print(f" [green] + {replacement[1].strip()}", highlight=False) - stderr.print("\n") - - with open(fn, "w") as fh: - fh.write(content) + if yaml_key: + update_yaml_file(fn, patterns, yaml_key, required) + else: + update_text_file(fn, patterns, required) + + +def update_yaml_file(fn: Path, patterns: List[Tuple[str, str]], yaml_key: List[str], required: bool): + import yaml + + with open(fn) as file: + yaml_content = yaml.safe_load(file) + + try: + target = yaml_content + for key in yaml_key[:-1]: + target = target[key] + + last_key = yaml_key[-1] + current_value = target[last_key] + + new_value = current_value + for pattern, replacement in patterns: + new_value = re.sub(pattern, replacement, new_value) + + if new_value != current_value: + target[last_key] = new_value + with open(fn, "w") as file: + yaml.dump(yaml_content, file) + log.info(f"Updated version in YAML file '{fn}'") + log_change(current_value, new_value) + except KeyError as e: + handle_error(f"Could not find key {e} in the YAML structure of {fn}", required) + + +def update_text_file(fn: Path, patterns: List[Tuple[str, str]], required: bool): + with open(fn) as file: + content = file.read() + + updated = False + for pattern, replacement in patterns: + new_content, count = re.subn(pattern, replacement, content) + if count > 0: + log_change(content, new_content) + content = new_content + updated = True + log.info(f"Updated version in '{fn}'") + log.debug(f"Replaced pattern '{pattern}' with '{replacement}' {count} times") + elif required: + handle_error(f"Could not find version number in {fn}: `{pattern}`", required) + + if updated: + with open(fn, "w") as file: + file.write(content) + + +def handle_error(message: str, required: bool): + if required: + raise ValueError(message) + else: + log.info(message) + + +def log_change(old_content: str, new_content: str): + old_lines = old_content.splitlines() + new_lines = new_content.splitlines() + + for old_line, new_line in zip(old_lines, new_lines): + if old_line != new_line: + stderr.print(f" [red] - {old_line.strip()}", highlight=False) + stderr.print(f" [green] + {new_line.strip()}", highlight=False) + + stderr.print("\n") From 41c4b95662ab4686aa39194d2ca5b427596e8815 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 13:06:48 +0200 Subject: [PATCH 05/10] add test for version bump in `.nf-core.yml` --- tests/pipelines/test_bump_version.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_bump_version.py b/tests/pipelines/test_bump_version.py index 709e82427d..8d6032f23f 100644 --- a/tests/pipelines/test_bump_version.py +++ b/tests/pipelines/test_bump_version.py @@ -13,12 +13,25 @@ def test_bump_pipeline_version(self): """Test that making a release with the working example files works""" # Bump the version number - nf_core.pipelines.bump_version.bump_pipeline_version(self.pipeline_obj, "1.1") + nf_core.pipelines.bump_version.bump_pipeline_version(self.pipeline_obj, "1.1.0") new_pipeline_obj = nf_core.utils.Pipeline(self.pipeline_dir) # Check nextflow.config new_pipeline_obj.load_pipeline_config() - assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.1" + assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.1.0" + + # Check multiqc_config.yml + with open(new_pipeline_obj._fp("assets/multiqc_config.yml")) as fh: + multiqc_config = yaml.safe_load(fh) + + assert "report_comment" in multiqc_config + assert "/releases/tag/1.1.0" in multiqc_config["report_comment"] + + # Check .nf-core.yml + with open(new_pipeline_obj._fp(".nf-core.yml")) as fh: + nf_core_yml = yaml.safe_load(fh) + if nf_core_yml["template"]: + assert nf_core_yml["template"]["version"] == "1.1.0" def test_dev_bump_pipeline_version(self): """Test that making a release works with a dev name and a leading v""" From 9e3d2d1cff1cdebeebf0f56c083c419474fbfeb8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 13:25:12 +0200 Subject: [PATCH 06/10] bump default nextflow version in test --- tests/pipelines/test_bump_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_bump_version.py b/tests/pipelines/test_bump_version.py index 8d6032f23f..408be5020b 100644 --- a/tests/pipelines/test_bump_version.py +++ b/tests/pipelines/test_bump_version.py @@ -46,7 +46,7 @@ def test_dev_bump_pipeline_version(self): def test_bump_nextflow_version(self): # Bump the version number to a specific version, preferably one # we're not already on - version = "22.04.3" + version = "24.04.2" nf_core.pipelines.bump_version.bump_nextflow_version(self.pipeline_obj, version) new_pipeline_obj = nf_core.utils.Pipeline(self.pipeline_dir) new_pipeline_obj._load() From 6e5d73782c6406cf98ae1bcac444e4e02f70cce6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Oct 2024 14:09:27 +0200 Subject: [PATCH 07/10] remove outdated readme pattern --- nf_core/pipelines/bump_version.py | 15 +++++++-------- tests/pipelines/test_bump_version.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index 55fb82ad20..b091fce70e 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -180,12 +180,7 @@ def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: ( rf"nextflow%20DSL2-%E2%89%A5{re.escape(current_version)}-23aa62.svg", f"nextflow%20DSL2-%E2%89%A5{new_version}-23aa62.svg", - ), - ( - # example: 1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=20.04.0`) - rf"1\.\s*Install\s*\[`Nextflow`\]\(https:\/\/www\.nextflow\.io\/docs\/latest\/getstarted\.html#installation\)\s*\(`>={re.escape(current_version)}`\)", - f"1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>={new_version}`)", - ), + ) ], ) @@ -225,14 +220,18 @@ def update_yaml_file(fn: Path, patterns: List[Tuple[str, str]], yaml_key: List[s new_value = current_value for pattern, replacement in patterns: - new_value = re.sub(pattern, replacement, new_value) + # check if current value is list + if isinstance(current_value, list): + new_value = [re.sub(pattern, replacement, item) for item in current_value] + else: + new_value = re.sub(pattern, replacement, current_value) if new_value != current_value: target[last_key] = new_value with open(fn, "w") as file: yaml.dump(yaml_content, file) log.info(f"Updated version in YAML file '{fn}'") - log_change(current_value, new_value) + log_change(str(current_value), str(new_value)) except KeyError as e: handle_error(f"Could not find key {e} in the YAML structure of {fn}", required) diff --git a/tests/pipelines/test_bump_version.py b/tests/pipelines/test_bump_version.py index 408be5020b..8af5c0e4d1 100644 --- a/tests/pipelines/test_bump_version.py +++ b/tests/pipelines/test_bump_version.py @@ -46,7 +46,7 @@ def test_dev_bump_pipeline_version(self): def test_bump_nextflow_version(self): # Bump the version number to a specific version, preferably one # we're not already on - version = "24.04.2" + version = "25.04.2" nf_core.pipelines.bump_version.bump_nextflow_version(self.pipeline_obj, version) new_pipeline_obj = nf_core.utils.Pipeline(self.pipeline_dir) new_pipeline_obj._load() From 634d7233a0d7f4e67b9f570b8659bdffcc35ad95 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 14 Oct 2024 12:11:07 +0000 Subject: [PATCH 08/10] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3a4eaa71d..1ceb2d6324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ ### General +- Include .nf-core.yml in `nf-core pipelines bump-version` ([#3220](https://github.com/nf-core/tools/pull/3220)) + ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] ### Template From 0efbd559cade59d2414d82d4b5938553e154ab22 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Oct 2024 10:12:03 +0200 Subject: [PATCH 09/10] keep yaml item order --- nf_core/pipelines/bump_version.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index b091fce70e..72246b8045 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -205,10 +205,13 @@ def update_file_version( def update_yaml_file(fn: Path, patterns: List[Tuple[str, str]], yaml_key: List[str], required: bool): - import yaml + from ruamel.yaml import YAML + + yaml = YAML() + yaml.preserve_quotes = True with open(fn) as file: - yaml_content = yaml.safe_load(file) + yaml_content = yaml.load(file) try: target = yaml_content From 30e3d5dfaaa427d77e83d28cb6a01874106fbe69 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Oct 2024 10:21:55 +0200 Subject: [PATCH 10/10] add docstrings --- nf_core/pipelines/bump_version.py | 33 +++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index 72246b8045..3190ed70d4 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -8,6 +8,7 @@ from typing import List, Optional, Tuple, Union import rich.console +from ruamel.yaml import YAML import nf_core.utils from nf_core.utils import Pipeline @@ -192,6 +193,18 @@ def update_file_version( required: bool = True, yaml_key: Optional[List[str]] = None, ) -> None: + """ + Updates a file with a new version number. + + Args: + filename (str): The name of the file to update. + pipeline_obj (nf_core.utils.Pipeline): A `Pipeline` object that holds information + about the pipeline contents. + patterns (List[Tuple[str, str]]): A list of tuples containing the regex patterns to + match and the replacement strings. + required (bool, optional): Whether the file is required to exist. Defaults to `True`. + yaml_key (Optional[List[str]], optional): The YAML key to update. Defaults to `None`. + """ fn: Path = pipeline_obj._fp(filename) if not fn.exists(): @@ -205,11 +218,18 @@ def update_file_version( def update_yaml_file(fn: Path, patterns: List[Tuple[str, str]], yaml_key: List[str], required: bool): - from ruamel.yaml import YAML + """ + Updates a YAML file with a new version number. + Args: + fn (Path): The name of the file to update. + patterns (List[Tuple[str, str]]): A list of tuples containing the regex patterns to + match and the replacement strings. + yaml_key (List[str]): The YAML key to update. + required (bool): Whether the file is required to exist. + """ yaml = YAML() yaml.preserve_quotes = True - with open(fn) as file: yaml_content = yaml.load(file) @@ -240,6 +260,15 @@ def update_yaml_file(fn: Path, patterns: List[Tuple[str, str]], yaml_key: List[s def update_text_file(fn: Path, patterns: List[Tuple[str, str]], required: bool): + """ + Updates a text file with a new version number. + + Args: + fn (Path): The name of the file to update. + patterns (List[Tuple[str, str]]): A list of tuples containing the regex patterns to + match and the replacement strings. + required (bool): Whether the file is required to exist. + """ with open(fn) as file: content = file.read()